test: ioverifier.c Source File

Main Page | Class Hierarchy | Class List | File List | Class Members | File Members

# ioverifier.c

Go to the documentation of this file.

    00001 #include "iop.h"
    00002
    00003 /*++
    00004
    00005 Copyright (c) 1989  Microsoft Corporation
    00006
    00007 Module Name:
    00008
    00009    ioverifier.c
    00010
    00011 Abstract:
    00012
    00013     This module contains the routines to verify suspect drivers.
    00014
    00015 Author:
    00016
    00017     Narayanan Ganapathy (narg) 8-Jan-1999
    00018
    00019 Revision History:
    00020
    00021     Adrian J. Oney (AdriaO) 28-Feb-1999
    00022         - merge in special irp code.
    00023
    00024 --*/
    00025
    00026 #if (( defined(_X86_) ) && ( FPO ))
    00027 #pragma optimize( "y", off )    // disable FPO for consistent stack traces
    00028 #endif
    00029
    00030
    00031 #define IO_FREE_IRP_TYPE_INVALID                1
    00032 #define IO_FREE_IRP_NOT_ASSOCIATED_WITH_THREAD  2
    00033 #define IO_CALL_DRIVER_IRP_TYPE_INVALID         3
    00034 #define IO_CALL_DRIVER_INVALID_DEVICE_OBJECT    4
    00035 #define IO_CALL_DRIVER_IRQL_NOT_EQUAL           5
    00036 #define IO_COMPLETE_REQUEST_INVALID_STATUS      6
    00037 #define IO_COMPLETE_REQUEST_CANCEL_ROUTINE_SET  7
    00038 #define IO_BUILD_FSD_REQUEST_EXCEPTION          8
    00039 #define IO_BUILD_IOCTL_REQUEST_EXCEPTION        9
    00040 #define IO_REINITIALIZING_TIMER_OBJECT          10
    00041 #define IO_INVALID_HANDLE                       11
    00042 #define IO_INVALID_STACK_IOSB                   12
    00043 #define IO_INVALID_STACK_EVENT                  13
    00044 #define IO_COMPLETE_REQUEST_INVALID_IRQL        14
    00045
    00046 //
    00047 // 0x200 and up are defined in ioassert.c
    00048 //
    00049
    00050 PVOID
    00051 VerifierAllocatePoolWithQuotaTag(
    00052     IN POOL_TYPE PoolType,
    00053     IN SIZE_T NumberOfBytes,
    00054     IN ULONG Tag
    00055     );
    00056
    00057 #define IsKernelHandle(H,M) (((LONG_PTR)(H) < 0) && ((M) == KernelMode) && ((H) != NtCurrentThread()) && ((H) != NtCurrentProcess()))
    00058
    00059 BOOLEAN
    00060 IovpValidateDeviceObject(
    00061     IN  PDEVICE_OBJECT  DeviceObject
    00062     );
    00063 VOID
    00064 IovFreeIrpPrivate(
    00065     IN  PIRP    Irp
    00066     );
    00067
    00068 #ifdef ALLOC_PRAGMA
    00069 #pragma alloc_text(INIT, IoVerifierInit)
    00070 #pragma alloc_text(PAGEVRFY,IovAllocateIrp)
    00071 #pragma alloc_text(PAGEVRFY,IovFreeIrp)
    00072 #pragma alloc_text(PAGEVRFY,IovCallDriver)
    00073 #pragma alloc_text(PAGEVRFY,IovCompleteRequest)
    00074 #pragma alloc_text(PAGEVRFY,IovCallDriver)
    00075 #pragma alloc_text(PAGEVRFY,IovCompleteRequest)
    00076 #pragma alloc_text(PAGEVRFY,IovpValidateDeviceObject)
    00077 #pragma alloc_text(PAGEVRFY,IovSpecialIrpCallDriver)
    00078 #pragma alloc_text(PAGEVRFY,IovSpecialIrpCompleteRequest)
    00079 #pragma alloc_text(PAGEVRFY,IovFreeIrpPrivate)
    00080 #endif
    00081
    00082 BOOLEAN         IopVerifierOn = FALSE;
    00083 ULONG           IovpEnforcementLevel = (ULONG) -1;
    00084 ULONG           IovpVerifierLevel = (ULONG)0;
    00085 BOOLEAN         IoVerifierOnByDefault = TRUE;
    00086 LONG            IovpInitCalled = 0;
    00087 ULONG           IovpMaxSupportedVerifierLevel = 3;
    00088 ULONG           IovpVerifierFlags = 0;               // Stashes the verifier flags passed at init.
    00089
    00090 VOID
    00091 IoVerifierInit(
    00092     IN ULONG VerifierFlags,
    00093     IN ULONG InitFlags
    00094     )
    00095 {
    00096     PVOID   sectionHeaderHandle;
    00097     ULONG   verifierLevel;
    00098
    00099     if (IoVerifierOnByDefault) {
    00100         VerifierFlags |= DRIVER_VERIFIER_IO_CHECKING;
    00101     }
    00102
    00103     if (!VerifierFlags) {
    00104         return;
    00105     }
    00106     pIoAllocateIrp = IovAllocateIrp;
    00107
    00108 #ifndef NO_SPECIAL_IRP
    00109     if (!(InitFlags & IOVERIFIERINIT_PHASE0)) {
    00110
    00111         //
    00112         // Lock it down.
    00113         //
    00114         sectionHeaderHandle = MmLockPagableCodeSection(IopDriverCorrectnessTakeLock);
    00115
    00116         if (!sectionHeaderHandle) {
    00117
    00118             return;
    00119         }
    00120     }
    00121
    00122     //
    00123     // Various initialization
    00124     //
    00125     if (IopDcControlOverride == (ULONG) -1) {
    00126
    00127         IopDcControlOverride = 0;
    00128     }
    00129
    00130     if (IopDcControlInitial == (ULONG) -1) {
    00131
    00132         IopDcControlInitial = 0;
    00133         if (!(InitFlags & IOVERIFIERINIT_VERIFIER_DRIVER_LIST)) {
    00134
    00135             IopDcControlInitial |= DIAG_IGNORE_DRIVER_LIST;
    00136         }
    00137     }
    00138
    00139     KeInitializeSpinLock(&IopDcControlLock) ;
    00140
    00141 #endif // NO_SPECIAL_IRP
    00142
    00143     if (!(VerifierFlags & DRIVER_VERIFIER_IO_CHECKING)) {
    00144
    00145         return;
    00146     }
    00147
    00148     //
    00149     // Determine the level of verification. Later we will modify the driver
    00150     // verifier applet to pass in a level directly.
    00151     //
    00152
    00153 #if !DBG
    00154     if (IovpVerifierLevel > IovpMaxSupportedVerifierLevel) {
    00155         IovpVerifierLevel  = IovpMaxSupportedVerifierLevel;
    00156     }
    00157 #endif
    00158
    00159     verifierLevel = IovpVerifierLevel;
    00160
    00161     //
    00162     // Enable and hook in the verifier.
    00163     //
    00164     IopVerifierOn = TRUE;
    00165     IovpInitCalled = 1;
    00166     IovpVerifierFlags = VerifierFlags;
    00167
    00168     if (verifierLevel > 1) {
    00169         //
    00170         // Initialize the special IRP code as appropriate.
    00171         //
    00172 #ifndef NO_SPECIAL_IRP
    00173         IovpInitIrpTracking(verifierLevel, InitFlags);
    00174 #endif // NO_SPECIAL_IRP
    00175         InterlockedExchangePointer((PVOID *)&pIofCallDriver, (PVOID) IovSpecialIrpCallDriver);
    00176         InterlockedExchangePointer((PVOID *)&pIofCompleteRequest, (PVOID) IovSpecialIrpCompleteRequest);
    00177         InterlockedExchangePointer((PVOID *)&pIoFreeIrp, (PVOID) IovFreeIrpPrivate);
    00178     }
    00179     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
    00180 }
    00181
    00182
    00183 BOOLEAN
    00184 IovpValidateDeviceObject(
    00185     IN  PDEVICE_OBJECT  DeviceObject
    00186     )
    00187 {
    00188     if ((DeviceObject->Type != IO_TYPE_DEVICE) ||
    00189         (DeviceObject->DriverObject == NULL) ||
    00190         (DeviceObject->ReferenceCount < 0 )) {
    00191         return FALSE;
    00192     } else {
    00193         return TRUE;
    00194     }
    00195 }
    00196
    00197 VOID
    00198 IovFreeIrp(
    00199     IN  PIRP    Irp
    00200     )
    00201 {
    00202     IovFreeIrpPrivate(Irp);
    00203 }
    00204
    00205 VOID
    00206 IovFreeIrpPrivate(
    00207     IN  PIRP    Irp
    00208     )
    00209 {
    00210
    00211
    00212 #ifndef NO_SPECIAL_IRP
    00213     BOOLEAN freeHandled ;
    00214 #endif
    00215
    00216
    00217     if (IopVerifierOn) {
    00218         if (Irp->Type != IO_TYPE_IRP) {
    00219             KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00220                          IO_FREE_IRP_TYPE_INVALID,
    00221                          (ULONG_PTR)Irp,
    00222                          0,
    00223                          0);
    00224         }
    00225         if (!IsListEmpty(&(Irp)->ThreadListEntry)) {
    00226             KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00227                          IO_FREE_IRP_NOT_ASSOCIATED_WITH_THREAD,
    00228                          (ULONG_PTR)Irp,
    00229                          0,
    00230                          0);
    00231         }
    00232     }
    00233
    00234 #ifndef NO_SPECIAL_IRP
    00235     SPECIALIRP_IO_FREE_IRP(Irp, &freeHandled) ;
    00236
    00237     if (freeHandled) {
    00238
    00239        return ;
    00240     }
    00241 #endif
    00242
    00243     IopFreeIrp(Irp);
    00244 }
    00245
    00246 NTSTATUS
    00247 FASTCALL
    00248 IovCallDriver(
    00249     IN  PDEVICE_OBJECT  DeviceObject,
    00250     IN  OUT PIRP    Irp
    00251     )
    00252 {
    00253     KIRQL    saveIrql;
    00254     NTSTATUS status;
    00255
    00256
    00257     if (!IopVerifierOn) {
    00258         status = IopfCallDriver(DeviceObject, Irp);
    00259         return status;
    00260     }
    00261
    00262     if (IovpVerifierLevel > 1) {
    00263         status = IovSpecialIrpCallDriver(DeviceObject, Irp);
    00264         return status;
    00265     }
    00266     if (Irp->Type != IO_TYPE_IRP) {
    00267         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00268                      IO_CALL_DRIVER_IRP_TYPE_INVALID,
    00269                      (ULONG_PTR)Irp,
    00270                      0,
    00271                      0);
    00272     }
    00273     if (!IovpValidateDeviceObject(DeviceObject)) {
    00274         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00275                      IO_CALL_DRIVER_INVALID_DEVICE_OBJECT,
    00276                      (ULONG_PTR)DeviceObject,
    00277                      0,
    00278                      0);
    00279     }
    00280
    00281     saveIrql = KeGetCurrentIrql();
    00282     status = IopfCallDriver(DeviceObject, Irp);
    00283     if (saveIrql != KeGetCurrentIrql()) {
    00284         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00285                      IO_CALL_DRIVER_IRQL_NOT_EQUAL,
    00286                      (ULONG_PTR)DeviceObject,
    00287                      saveIrql,
    00288                      KeGetCurrentIrql());
    00289
    00290     }
    00291     return status;
    00292 }
    00293
    00294
    00295 VOID
    00296 FASTCALL
    00297 IovCompleteRequest(
    00298     IN  PIRP    Irp,
    00299     IN  CCHAR   PriorityBoost
    00300     )
    00301 {
    00302
    00303     if (!IopVerifierOn) {
    00304         IopfCompleteRequest(Irp, PriorityBoost);
    00305         return;
    00306     }
    00307
    00308     if (IovpVerifierLevel > 1) {
    00309         IovSpecialIrpCompleteRequest(Irp, PriorityBoost);
    00310         return;
    00311     }
    00312
    00313     if (Irp->CurrentLocation > (CCHAR) (Irp->StackCount + 1) ||
    00314         Irp->Type != IO_TYPE_IRP) {
    00315         KeBugCheckEx( MULTIPLE_IRP_COMPLETE_REQUESTS,
    00316                       (ULONG_PTR) Irp,
    00317                       __LINE__,
    00318                       0,
    00319                       0);
    00320     }
    00321
    00322     if (Irp->CancelRoutine) {
    00323         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00324                      IO_COMPLETE_REQUEST_CANCEL_ROUTINE_SET,
    00325                      (ULONG_PTR)Irp->CancelRoutine,
    00326                      (ULONG_PTR)Irp,
    00327                      0);
    00328     }
    00329     if (Irp->IoStatus.Status == STATUS_PENDING || Irp->IoStatus.Status == 0xffffffff) {
    00330          KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00331                       IO_COMPLETE_REQUEST_INVALID_STATUS,
    00332                       Irp->IoStatus.Status,
    00333                       (ULONG_PTR)Irp,
    00334                       0);
    00335     }
    00336     if (KeGetCurrentIrql() > DISPATCH_LEVEL) {
    00337         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00338                      IO_COMPLETE_REQUEST_INVALID_IRQL,
    00339                      KeGetCurrentIrql(),
    00340                      (ULONG_PTR)Irp,
    00341                      0);
    00342
    00343     }
    00344     IopfCompleteRequest(Irp, PriorityBoost);
    00345 }
    00346
    00347
    00348 //
    00349 // Wrapper for IovAllocateIrp. Use special pool to allocate the IRP.
    00350 // This is directly called from IoAllocateIrp.
    00351 //
    00352 PIRP
    00353 IovAllocateIrp(
    00354     IN CCHAR StackSize,
    00355     IN BOOLEAN ChargeQuota
    00356     )
    00357 {
    00358     USHORT allocateSize;
    00359     UCHAR fixedSize;
    00360     PIRP irp;
    00361     UCHAR mustSucceed;
    00362     USHORT packetSize;
    00363
    00364 #ifndef NO_SPECIAL_IRP
    00365     //
    00366     // Should we override normal lookaside caching so that we may catch
    00367     // more bugs?
    00368     //
    00369     SPECIALIRP_IO_ALLOCATE_IRP_1(StackSize, ChargeQuota, &irp) ;
    00370
    00371     if (irp) {
    00372        return irp ;
    00373     }
    00374 #endif
    00375
    00376     //
    00377     // If special pool is not turned on lets just call the standard
    00378     // irp allocator.
    00379     //
    00380
    00381     if (!(IovpVerifierFlags & DRIVER_VERIFIER_SPECIAL_POOLING )) {
    00382         irp = IopAllocateIrpPrivate(StackSize, ChargeQuota);
    00383         return irp;
    00384     }
    00385
    00386
    00387     irp = NULL;
    00388     fixedSize = 0;
    00389     mustSucceed = 0;
    00390     packetSize = IoSizeOfIrp(StackSize);
    00391     allocateSize = packetSize;
    00392
    00393     //
    00394     // There are no free packets on the lookaside list, or the packet is
    00395     // too large to be allocated from one of the lists, so it must be
    00396     // allocated from nonpaged pool. If quota is to be charged, charge it
    00397     // against the current process. Otherwise, allocate the pool normally.
    00398     //
    00399
    00400     if (ChargeQuota) {
    00401         try {
    00402             irp = ExAllocatePoolWithTagPriority(
    00403                     NonPagedPool,
    00404                     allocateSize,
    00405                     ' prI',
    00406                     HighPoolPrioritySpecialPoolOverrun);
    00407         } except(EXCEPTION_EXECUTE_HANDLER) {
    00408             NOTHING;
    00409         }
    00410
    00411     } else {
    00412
    00413         //
    00414         // Attempt to allocate the pool from non-paged pool.  If this
    00415         // fails, and the caller's previous mode was kernel then allocate
    00416         // the pool as must succeed.
    00417         //
    00418
    00419         irp = ExAllocatePoolWithTagPriority(
    00420                 NonPagedPool,
    00421                 allocateSize,
    00422                 ' prI',
    00423                 HighPoolPrioritySpecialPoolOverrun);
    00424         if (!irp) {
    00425             mustSucceed = IRP_ALLOCATED_MUST_SUCCEED;
    00426             if (KeGetPreviousMode() == KernelMode ) {
    00427                 irp = ExAllocatePoolWithTagPriority(
    00428                         NonPagedPoolMustSucceed,
    00429                         allocateSize,
    00430                         ' prI',
    00431                         HighPoolPrioritySpecialPoolOverrun);
    00432             }
    00433         }
    00434     }
    00435
    00436     if (!irp) {
    00437         return NULL;
    00438     }
    00439
    00440     //
    00441     // Initialize the packet.
    00442     //
    00443
    00444     IopInitializeIrp(irp, packetSize, StackSize);
    00445     irp->AllocationFlags = mustSucceed;
    00446     if (ChargeQuota) {
    00447         irp->AllocationFlags |= IRP_QUOTA_CHARGED;
    00448     }
    00449
    00450     SPECIALIRP_IO_ALLOCATE_IRP_2(irp) ;
    00451     return irp;
    00452 }
    00453
    00454
    00455 PIRP
    00456 IovBuildAsynchronousFsdRequest(
    00457     IN ULONG MajorFunction,
    00458     IN PDEVICE_OBJECT DeviceObject,
    00459     IN OUT PVOID Buffer OPTIONAL,
    00460     IN ULONG Length OPTIONAL,
    00461     IN PLARGE_INTEGER StartingOffset OPTIONAL,
    00462     IN PIO_STATUS_BLOCK IoStatusBlock OPTIONAL
    00463     )
    00464 {
    00465     PIRP    Irp;
    00466
    00467     try {
    00468         Irp = IoBuildAsynchronousFsdRequest(
    00469             MajorFunction,
    00470             DeviceObject,
    00471             Buffer,
    00472             Length,
    00473             StartingOffset,
    00474             IoStatusBlock
    00475             );
    00476     } except(EXCEPTION_EXECUTE_HANDLER) {
    00477          KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00478                       IO_BUILD_FSD_REQUEST_EXCEPTION,
    00479                       (ULONG_PTR)DeviceObject,
    00480                       (ULONG_PTR)MajorFunction,
    00481                       GetExceptionCode());
    00482     }
    00483     return (Irp);
    00484 }
    00485
    00486 PIRP
    00487 IovBuildDeviceIoControlRequest(
    00488     IN ULONG IoControlCode,
    00489     IN PDEVICE_OBJECT DeviceObject,
    00490     IN PVOID InputBuffer OPTIONAL,
    00491     IN ULONG InputBufferLength,
    00492     OUT PVOID OutputBuffer OPTIONAL,
    00493     IN ULONG OutputBufferLength,
    00494     IN BOOLEAN InternalDeviceIoControl,
    00495     IN PKEVENT Event,
    00496     OUT PIO_STATUS_BLOCK IoStatusBlock
    00497     )
    00498 {
    00499     PIRP    Irp;
    00500
    00501     try {
    00502         Irp = IoBuildDeviceIoControlRequest(
    00503             IoControlCode,
    00504             DeviceObject,
    00505             InputBuffer,
    00506             InputBufferLength,
    00507             OutputBuffer,
    00508             OutputBufferLength,
    00509             InternalDeviceIoControl,
    00510             Event,
    00511             IoStatusBlock
    00512             );
    00513     } except(EXCEPTION_EXECUTE_HANDLER) {
    00514          KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00515                       IO_BUILD_IOCTL_REQUEST_EXCEPTION,
    00516                       (ULONG_PTR)DeviceObject,
    00517                       (ULONG_PTR)IoControlCode,
    00518                       GetExceptionCode());
    00519     }
    00520
    00521     return (Irp);
    00522 }
    00523
    00524 NTSTATUS
    00525 IovInitializeTimer(
    00526     IN PDEVICE_OBJECT DeviceObject,
    00527     IN PIO_TIMER_ROUTINE TimerRoutine,
    00528     IN PVOID Context
    00529     )
    00530 {
    00531     if (DeviceObject->Timer) {
    00532         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00533                      IO_REINITIALIZING_TIMER_OBJECT,
    00534                      (ULONG_PTR)DeviceObject,
    00535                      0,
    00536                      0);
    00537    }
    00538    return (IoInitializeTimer(DeviceObject, TimerRoutine, Context));
    00539 }
    00540
    00541
    00542 VOID
    00543 IovpCompleteRequest(
    00544     IN PKAPC Apc,
    00545     IN PVOID *SystemArgument1,
    00546     IN PVOID *SystemArgument2
    00547     )
    00548 {
    00549     PIRP    irp;
    00550     PUCHAR addr;
    00551     ULONG   BestStackOffset;
    00552
    00553     irp = CONTAINING_RECORD( Apc, IRP, Tail.Apc );
    00554
    00555 #if defined(_X86_)
    00556
    00557
    00558     addr = (PUCHAR)irp->UserIosb;
    00559     if ((addr > (PUCHAR)KeGetCurrentThread()->StackLimit) &&
    00560         (addr <= (PUCHAR)&BestStackOffset)) {
    00561         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00562                      IO_INVALID_STACK_IOSB,
    00563                      (ULONG_PTR)addr,
    00564                      0,
    00565                      0);
    00566
    00567     }
    00568
    00569     addr = (PUCHAR)irp->UserEvent;
    00570     if ((addr > (PUCHAR)KeGetCurrentThread()->StackLimit) &&
    00571         (addr <= (PUCHAR)&BestStackOffset)) {
    00572         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00573                      IO_INVALID_STACK_EVENT,
    00574                      (ULONG_PTR)addr,
    00575                      0,
    00576                      0);
    00577
    00578     }
    00579 #endif
    00580 }
    00581
    00582
    00583 /*-------------------------- SPECIALIRP HOOKS -------------------------------*/
    00584
    00585 VOID
    00586 FASTCALL
    00587 IovSpecialIrpCompleteRequest(
    00588     IN PIRP Irp,
    00589     IN CCHAR PriorityBoost
    00590     )
    00591 /*++
    00592
    00593 Routine Description:
    00594
    00595     The following code is called only when doing IRP tracking. It duplicates code
    00596     in iosubs.c.
    00597     This routine is invoked to complete an I/O request.  It is invoked by the
    00598     driver in its DPC routine to perform the final completion of the IRP.  The
    00599     functions performed by this routine are as follows.
    00600
    00601         1.  A check is made to determine whether the packet's stack locations
    00602             have been exhausted.  If not, then the stack location pointer is set
    00603             to the next location and if there is a routine to be invoked, then
    00604             it will be invoked.  This continues until there are either no more
    00605             routines which are interested or the packet runs out of stack.
    00606
    00607             If a routine is invoked to complete the packet for a specific driver
    00608             which needs to perform work a lot of work or the work needs to be
    00609             performed in the context of another process, then the routine will
    00610             return an alternate success code of STATUS_MORE_PROCESSING_REQUIRED.
    00611             This indicates that this completion routine should simply return to
    00612             its caller because the operation will be "completed" by this routine
    00613             again sometime in the future.
    00614
    00615         2.  A check is made to determine whether this IRP is an associated IRP.
    00616             If it is, then the count on the master IRP is decremented.  If the
    00617             count for the master becomes zero, then the master IRP will be
    00618             completed according to the steps below taken for a normal IRP being
    00619             completed.  If the count is still non-zero, then this IRP (the one
    00620             being completed) will simply be deallocated.
    00621
    00622         3.  If this is paging I/O or a close operation, then simply write the
    00623             I/O status block and set the event to the signaled state, and
    00624             dereference the event.  If this is paging I/O, deallocate the IRP
    00625             as well.
    00626
    00627         4.  Unlock the pages, if any, specified by the MDL by calling
    00628             MmUnlockPages.
    00629
    00630         5.  A check is made to determine whether or not completion of the
    00631             request can be deferred until later.  If it can be, then this
    00632             routine simply exits and leaves it up to the originator of the
    00633             request to fully complete the IRP.  By not initializing and queueing
    00634             the special kernel APC to the calling thread (which is the current
    00635             thread by definition), a lot of interrupt and queueing processing
    00636             can be avoided.
    00637
    00638
    00639         6.  The final rundown routine is invoked to queue the request packet to
    00640             the target (requesting) thread as a special kernel mode APC.
    00641
    00642 Arguments:
    00643
    00644     Irp - Pointer to the I/O Request Packet to complete.
    00645
    00646     PriorityBoost - Supplies the amount of priority boost that is to be given
    00647         to the target thread when the special kernel APC is queued.
    00648
    00649 Return Value:
    00650
    00651     None.
    00652
    00653 --*/
    00654
    00655 #define ZeroAndDopeIrpStackLocation( IrpSp ) {  \
    00656     (IrpSp)->MinorFunction = 0;                 \
    00657     (IrpSp)->Flags = 0;                         \
    00658     (IrpSp)->Control = SL_NOTCOPIED;            \
    00659     (IrpSp)->Parameters.Others.Argument1 = 0;   \
    00660     (IrpSp)->Parameters.Others.Argument2 = 0;   \
    00661     (IrpSp)->Parameters.Others.Argument3 = 0;   \
    00662     (IrpSp)->Parameters.Others.Argument4 = 0;   \
    00663     (IrpSp)->FileObject = (PFILE_OBJECT) NULL; }
    00664
    00665 {
    00666     PIRP masterIrp;
    00667     NTSTATUS status;
    00668     PIO_STACK_LOCATION stackPointer;
    00669     PMDL mdl;
    00670     PETHREAD thread;
    00671     PFILE_OBJECT fileObject;
    00672     KIRQL irql;
    00673     PVOID saveAuxiliaryPointer = NULL;
    00674
    00675 #ifndef NO_SPECIAL_IRP
    00676     PVOID routine ;
    00677     IOFCOMPLETEREQUEST_STACKDATA completionPacket;
    00678 #endif
    00679
    00680     if (!IopVerifierOn) {
    00681         IopfCompleteRequest(Irp, PriorityBoost);
    00682         return;
    00683     }
    00684
    00685 #if DBG
    00686
    00687     if (Irp->CurrentLocation <= (CCHAR) Irp->StackCount) {
    00688
    00689         stackPointer = IoGetCurrentIrpStackLocation(Irp);
    00690         if (stackPointer->MajorFunction == IRP_MJ_POWER) {
    00691             PoPowerTrace(
    00692                 POWERTRACE_COMPLETE,
    00693                 IoGetCurrentIrpStackLocation(Irp)->DeviceObject,
    00694                 Irp,
    00695                 IoGetCurrentIrpStackLocation(Irp)
    00696                 );
    00697         }
    00698     }
    00699
    00700 #endif
    00701
    00702     SPECIALIRP_IOF_COMPLETE_1(Irp, PriorityBoost, &completionPacket);
    00703
    00704     //
    00705     // Begin by ensuring that this packet has not already been completed
    00706     // by someone.
    00707     //
    00708
    00709     if (Irp->CurrentLocation > (CCHAR) (Irp->StackCount + 1) ||
    00710         Irp->Type != IO_TYPE_IRP) {
    00711         KeBugCheckEx( MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR) Irp, __LINE__, 0, 0 );
    00712     }
    00713
    00714     //
    00715     // Ensure that the packet being completed really is still an IRP.
    00716     //
    00717
    00718     if (Irp->Type != IO_TYPE_IRP) {
    00719
    00720         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00721                      IO_CALL_DRIVER_IRP_TYPE_INVALID,
    00722                      (ULONG_PTR)Irp,
    00723                      0,
    00724                      0);
    00725     }
    00726
    00727     //
    00728     // Ensure that no one believes that this request is still in a cancellable
    00729     // state.
    00730     //
    00731
    00732     if (Irp->CancelRoutine) {
    00733
    00734         ASSERT(Irp->CancelRoutine == NULL);
    00735
    00736         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00737                      IO_COMPLETE_REQUEST_CANCEL_ROUTINE_SET,
    00738                      (ULONG_PTR)Irp->CancelRoutine,
    00739                      (ULONG_PTR)Irp,
    00740                      0);
    00741     }
    00742
    00743     //
    00744     // Ensure that the packet is not being completed with a thoroughly
    00745     // confusing status code.  Actually completing a packet with a pending
    00746     // status probably means that someone forgot to set the real status in
    00747     // the packet.
    00748
    00749     if (Irp->IoStatus.Status == STATUS_PENDING) {
    00750
    00751
    00752          KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00753                       IO_COMPLETE_REQUEST_INVALID_STATUS,
    00754                       Irp->IoStatus.Status,
    00755                       (ULONG_PTR)Irp,
    00756                       0);
    00757     }
    00758
    00759     //
    00760     // Ensure that the packet is not being completed with a minus one.  This is
    00761     // apparently a common problem in some drivers, and has no meaning as a
    00762     // status code.
    00763     //
    00764
    00765     if (Irp->IoStatus.Status == 0xffffffff) {
    00766
    00767
    00768          KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    00769                       IO_COMPLETE_REQUEST_INVALID_STATUS,
    00770                       Irp->IoStatus.Status,
    00771                       (ULONG_PTR)Irp,
    00772                       0);
    00773     }
    00774
    00775     //
    00776     // Ensure that if this is a paging I/O operation, and it failed, that the
    00777     // reason for the failure isn't because quota was exceeded.
    00778     //
    00779
    00780     ASSERT( !(Irp->Flags & IRP_PAGING_IO && Irp->IoStatus.Status == STATUS_QUOTA_EXCEEDED ) );
    00781
    00782 #ifndef NO_SPECIAL_IRP
    00783
    00784     if (!IovpTrackingFlags) {
    00785
    00786         IopfCompleteRequest(Irp, PriorityBoost);
    00787         return;
    00788     }
    00789
    00790     //
    00791     // Now check to see whether this is the last driver that needs to be
    00792     // invoked for this packet.  If not, then bump the stack and check to
    00793     // see whether the driver wishes to see the completion.  As each stack
    00794     // location is examined, invoke any routine which needs to be invoked.
    00795     // If the routine returns STATUS_MORE_PROCESSING_REQUIRED, then stop the
    00796     // processing of this packet.
    00797     //
    00798
    00799     for (stackPointer = IoGetCurrentIrpStackLocation( Irp ),
    00800          Irp->CurrentLocation++,
    00801          Irp->Tail.Overlay.CurrentStackLocation++;
    00802          Irp->CurrentLocation <= (CCHAR) (Irp->StackCount + 1);
    00803          stackPointer++,
    00804          Irp->CurrentLocation++,
    00805          Irp->Tail.Overlay.CurrentStackLocation++) {
    00806
    00807         //
    00808         // A stack location was located.  Check to see whether or not it
    00809         // has a completion routine and if so, whether or not it should be
    00810         // invoked.
    00811         //
    00812         // Begin by saving the pending returned flag in the current stack
    00813         // location in the fixed part of the IRP.
    00814         //
    00815
    00816         Irp->PendingReturned = stackPointer->Control & SL_PENDING_RETURNED;
    00817
    00818         SPECIALIRP_IOF_COMPLETE_2(Irp, &completionPacket);
    00819
    00820         if ( (NT_SUCCESS( Irp->IoStatus.Status ) &&
    00821              stackPointer->Control & SL_INVOKE_ON_SUCCESS) ||
    00822              (!NT_SUCCESS( Irp->IoStatus.Status ) &&
    00823              stackPointer->Control & SL_INVOKE_ON_ERROR) ||
    00824              (Irp->Cancel &&
    00825              stackPointer->Control & SL_INVOKE_ON_CANCEL)
    00826            ) {
    00827
    00828             //
    00829             // This driver has specified a completion routine.  Invoke the
    00830             // routine passing it a pointer to its device object and the
    00831             // IRP that is being completed.
    00832             //
    00833
    00834             ZeroAndDopeIrpStackLocation( stackPointer );
    00835
    00836 #ifndef NO_SPECIAL_IRP
    00837             routine = stackPointer->CompletionRoutine ;
    00838             SPECIALIRP_IOF_COMPLETE_3(Irp, routine, &completionPacket);
    00839 #endif
    00840
    00841             PERFINFO_DRIVER_COMPLETIONROUTINE_CALL(Irp, stackPointer);
    00842
    00843             status = stackPointer->CompletionRoutine( (PDEVICE_OBJECT) (Irp->CurrentLocation == (CCHAR) (Irp->StackCount + 1) ?
    00844                                                       (PDEVICE_OBJECT) NULL :
    00845                                                       IoGetCurrentIrpStackLocation( Irp )->DeviceObject),
    00846                                                       Irp,
    00847                                                       stackPointer->Context );
    00848
    00849             PERFINFO_DRIVER_COMPLETIONROUTINE_RETURN(Irp, stackPointer);
    00850
    00851             SPECIALIRP_IOF_COMPLETE_4(Irp, status, &completionPacket);
    00852
    00853             if (status == STATUS_MORE_PROCESSING_REQUIRED) {
    00854
    00855                 //
    00856                 // Note:  Notice that if the driver has returned the above
    00857                 //        status value, it may have already DEALLOCATED the
    00858                 //        packet!  Therefore, do NOT touch any part of the
    00859                 //        IRP in the following code.
    00860                 //
    00861
    00862                 SPECIALIRP_IOF_COMPLETE_5(Irp, &completionPacket);
    00863                 return;
    00864             }
    00865
    00866         } else {
    00867             if (Irp->PendingReturned && Irp->CurrentLocation <= Irp->StackCount) {
    00868                 IoMarkIrpPending( Irp );
    00869             }
    00870             ZeroAndDopeIrpStackLocation( stackPointer );
    00871         }
    00872
    00873         SPECIALIRP_IOF_COMPLETE_5(Irp, &completionPacket);
    00874     }
    00875
    00876     //
    00877     // Check to see whether this is an associated IRP.  If so, then decrement
    00878     // the count in the master IRP.  If the count is decremented to zero,
    00879     // then complete the master packet as well.
    00880     //
    00881
    00882     if (Irp->Flags & IRP_ASSOCIATED_IRP) {
    00883         ULONG count;
    00884         masterIrp = Irp->AssociatedIrp.MasterIrp;
    00885         count = ExInterlockedAddUlong( (PULONG) &masterIrp->AssociatedIrp.IrpCount,
    00886                                        0xffffffff,
    00887                                        &IopDatabaseLock );
    00888
    00889         //
    00890         // Deallocate this packet and any MDLs that are associated with it
    00891         // by either doing direct deallocations if they were allocated from
    00892         // a zone or by queueing the packet to a thread to perform the
    00893         // deallocation.
    00894         //
    00895         // Also, check the count of the master IRP to determine whether or not
    00896         // the count has gone to zero.  If not, then simply get out of here.
    00897         // Otherwise, complete the master packet.
    00898         //
    00899
    00900         Irp->Tail.Overlay.Thread = masterIrp->Tail.Overlay.Thread;
    00901         IopFreeIrpAndMdls( Irp );
    00902         if (count == 1) {
    00903             IoCompleteRequest( masterIrp, PriorityBoost );
    00904         }
    00905         return;
    00906     }
    00907
    00908     //
    00909     // Check to see if we have a name junction. If so set the stage to
    00910     // transmogrify the reparse point data in IopCompleteRequest.
    00911     //
    00912
    00913     if ((Irp->IoStatus.Status == STATUS_REPARSE )  &&
    00914         (Irp->IoStatus.Information > IO_REPARSE_TAG_RESERVED_RANGE)) {
    00915
    00916         if (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT) {
    00917
    00918             //
    00919             // For name junctions, we save the pointer to the auxiliary
    00920             // buffer and use it below.
    00921             //
    00922
    00923             ASSERT( Irp->Tail.Overlay.AuxiliaryBuffer != NULL );
    00924
    00925             saveAuxiliaryPointer = (PVOID) Irp->Tail.Overlay.AuxiliaryBuffer;
    00926
    00927             //
    00928             // We NULL the entry to avoid its de-allocation at this time.
    00929             // This buffer get deallocated in IopDoNameTransmogrify
    00930             //
    00931
    00932             Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
    00933         } else {
    00934
    00935             //
    00936             // Fail the request. A driver needed to act on this IRP prior
    00937             // to getting to this point.
    00938             //
    00939
    00940             Irp->IoStatus.Status = STATUS_IO_REPARSE_TAG_NOT_HANDLED;
    00941         }
    00942     }
    00943
    00944     //
    00945     // Check the auxiliary buffer pointer in the packet and if a buffer was
    00946     // allocated, deallocate it now.  Note that this buffer must be freed
    00947     // here since the pointer is overlayed with the APC that will be used
    00948     // to get to the requesting thread's context.
    00949     //
    00950
    00951     if (Irp->Tail.Overlay.AuxiliaryBuffer) {
    00952         ExFreePool( Irp->Tail.Overlay.AuxiliaryBuffer );
    00953         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
    00954     }
    00955
    00956     //
    00957     // Check to see if this is paging I/O or a close operation.  If either,
    00958     // then special processing must be performed.  The reasons that special
    00959     // processing must be performed is different based on the type of
    00960     // operation being performed.  The biggest reasons for special processing
    00961     // on paging operations are that using a special kernel APC for an in-
    00962     // page operation cannot work since the special kernel APC can incur
    00963     // another pagefault.  Likewise, all paging I/O uses MDLs that belong
    00964     // to the memory manager, not the I/O system.
    00965     //
    00966     // Close operations are special because the close may have been invoked
    00967     // because of a special kernel APC (some IRP was completed which caused
    00968     // the reference count on the object to become zero while in the I/O
    00969     // system's special kernel APC routine).  Therefore, a special kernel APC
    00970     // cannot be used since it cannot execute until the close APC finishes.
    00971     //
    00972     // The special steps are as follows for a synchronous paging operation
    00973     // and close are:
    00974     //
    00975     //     1.  Copy the I/O status block (it is in SVAS, nonpaged).
    00976     //     2.  Signal the event
    00977     //     3.  If paging I/O, deallocate the IRP
    00978     //
    00979     // The special steps taken for asynchronous paging operations (out-pages)
    00980     // are as follows:
    00981     //
    00982     //     1.  Initialize a special kernel APC just for page writes.
    00983     //     1.  Queue the special kernel APC.
    00984     //
    00985     // It should also be noted that the logic for completing a Mount request
    00986     // operation is exactly the same as a Page Read.  No assumptions should be
    00987     // made here about this being a Page Read operation w/o carefully checking
    00988     // to ensure that they are also true for a Mount.  That is:
    00989     //
    00990     //     IRP_PAGING_IO  and  IRP_MOUNT_COMPLETION
    00991     //
    00992     // are the same flag in the IRP.
    00993     //
    00994     // Also note that the last time the IRP is touched for a close operation
    00995     // must be just before the event is set to the signaled state.  Once this
    00996     // occurs, the IRP can be deallocated by the thread waiting for the event.
    00997     //
    00998
    00999     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION)) {
    01000         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION)) {
    01001             ULONG flags;
    01002
    01003             flags = Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO;
    01004             *Irp->UserIosb = Irp->IoStatus;
    01005             (VOID) KeSetEvent( Irp->UserEvent, PriorityBoost, FALSE );
    01006             if (flags) {
    01007                 IoFreeIrp( Irp );
    01008             }
    01009         } else {
    01010             thread = Irp->Tail.Overlay.Thread;
    01011             KeInitializeApc( &Irp->Tail.Apc,
    01012                              &thread->Tcb,
    01013                              Irp->ApcEnvironment,
    01014                              IopCompletePageWrite,
    01015                              (PKRUNDOWN_ROUTINE) NULL,
    01016                              (PKNORMAL_ROUTINE) NULL,
    01017                              KernelMode,
    01018                              (PVOID) NULL );
    01019             (VOID) KeInsertQueueApc( &Irp->Tail.Apc,
    01020                                      (PVOID) NULL,
    01021                                      (PVOID) NULL,
    01022                                      PriorityBoost );
    01023         }
    01024         return;
    01025     }
    01026
    01027     //
    01028     // Check to see whether any pages need to be unlocked.
    01029     //
    01030
    01031     if (Irp->MdlAddress != NULL) {
    01032
    01033         //
    01034         // Unlock any pages that may be described by MDLs.
    01035         //
    01036
    01037         mdl = Irp->MdlAddress;
    01038         while (mdl != NULL) {
    01039             MmUnlockPages( mdl );
    01040             mdl = mdl->Next;
    01041         }
    01042     }
    01043
    01044     //
    01045     // Make a final check here to determine whether or not this is a
    01046     // synchronous I/O operation that is being completed in the context
    01047     // of the original requestor.  If so, then an optimal path through
    01048     // I/O completion can be taken.
    01049     //
    01050
    01051     if (Irp->Flags & IRP_DEFER_IO_COMPLETION && !Irp->PendingReturned) {
    01052
    01053         if ((Irp->IoStatus.Status == STATUS_REPARSE )  &&
    01054             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT)) {
    01055
    01056             //
    01057             // For name junctions we reinstate the address of the appropriate
    01058             // buffer. It is freed in parse.c
    01059             //
    01060
    01061             Irp->Tail.Overlay.AuxiliaryBuffer = saveAuxiliaryPointer;
    01062         }
    01063
    01064         return;
    01065     }
    01066
    01067     //
    01068     // Finally, initialize the IRP as an APC structure and queue the special
    01069     // kernel APC to the target thread.
    01070     //
    01071
    01072     thread = Irp->Tail.Overlay.Thread;
    01073     fileObject = Irp->Tail.Overlay.OriginalFileObject;
    01074
    01075     if (!Irp->Cancel) {
    01076
    01077         KeInitializeApc( &Irp->Tail.Apc,
    01078                          &thread->Tcb,
    01079                          Irp->ApcEnvironment,
    01080                          IopCompleteRequest,
    01081                          IopAbortRequest,
    01082                          (PKNORMAL_ROUTINE) NULL,
    01083                          KernelMode,
    01084                          (PVOID) NULL );
    01085
    01086         (VOID) KeInsertQueueApc( &Irp->Tail.Apc,
    01087                                  fileObject,
    01088                                  (PVOID) saveAuxiliaryPointer,
    01089                                  PriorityBoost );
    01090     } else {
    01091
    01092         //
    01093         // This request has been cancelled.  Ensure that access to the thread
    01094         // is synchronized, otherwise it may go away while attempting to get
    01095         // through the remainder of completion for this request.  This happens
    01096         // when the thread times out waiting for the request to be completed
    01097         // once it has been cancelled.
    01098         //
    01099         // Note that it is safe to capture the thread pointer above, w/o having
    01100         // the lock because the cancel flag was not set at that point, and
    01101         // the code that disassociates IRPs must set the flag before looking to
    01102         // see whether or not the packet has been completed, and this packet
    01103         // will appear to be completed because it no longer belongs to a driver.
    01104         //
    01105
    01106         ExAcquireSpinLock( &IopCompletionLock, &irql );
    01107
    01108         thread = Irp->Tail.Overlay.Thread;
    01109
    01110         if (thread) {
    01111
    01112             KeInitializeApc( &Irp->Tail.Apc,
    01113                              &thread->Tcb,
    01114                              Irp->ApcEnvironment,
    01115                              IopCompleteRequest,
    01116                              IopAbortRequest,
    01117                              (PKNORMAL_ROUTINE) NULL,
    01118                              KernelMode,
    01119                              (PVOID) NULL );
    01120
    01121             (VOID) KeInsertQueueApc( &Irp->Tail.Apc,
    01122                                      fileObject,
    01123                                      (PVOID) saveAuxiliaryPointer,
    01124                                      PriorityBoost );
    01125
    01126             ExReleaseSpinLock( &IopCompletionLock, irql );
    01127
    01128         } else {
    01129
    01130             //
    01131             // This request has been aborted from completing in the caller's
    01132             // thread.  This can only occur if the packet was cancelled, and
    01133             // the driver did not complete the request, so it was timed out.
    01134             // Attempt to drop things on the floor, since the originating thread
    01135             // has probably exited at this point.
    01136             //
    01137
    01138             ExReleaseSpinLock( &IopCompletionLock, irql );
    01139
    01140             ASSERT( Irp->Cancel );
    01141
    01142             //
    01143             // Drop the IRP on the floor.
    01144             //
    01145
    01146             IopDropIrp( Irp, fileObject );
    01147
    01148         }
    01149     }
    01150 #else
    01151
    01152     IopfCompleteRequest(Irp, PriorityBoost);
    01153     return;
    01154
    01155 #endif
    01156 }
    01157
    01158 NTSTATUS
    01159 FASTCALL
    01160 IovSpecialIrpCallDriver(
    01161     IN  PDEVICE_OBJECT  DeviceObject,
    01162     IN  OUT PIRP    Irp
    01163     )
    01164 /*++
    01165
    01166 Routine Description:
    01167
    01168     This routine is invoked to pass an I/O Request Packet (IRP) to another
    01169     driver at its dispatch routine. This routine is called only for IRP tracking
    01170     It duplicates the code in iosubs.c.
    01171
    01172 Arguments:
    01173
    01174     DeviceObject - Pointer to device object to which the IRP should be passed.
    01175
    01176     Irp - Pointer to IRP for request.
    01177
    01178 Return Value:
    01179
    01180     Return status from driver's dispatch routine.
    01181
    01182 --*/
    01183
    01184 {
    01185     PIO_STACK_LOCATION irpSp;
    01186     PDRIVER_OBJECT driverObject;
    01187     NTSTATUS status;
    01188     KIRQL saveIrql;
    01189     PDRIVER_DISPATCH dispatchRoutine;
    01190     IOFCALLDRIVER_STACKDATA iofCallDriverStackData ;
    01191
    01192     if (!IopVerifierOn) {
    01193         status = IopfCallDriver(DeviceObject, Irp);
    01194         return status;
    01195     }
    01196
    01197     //
    01198     // Ensure that this is really an I/O Request Packet.
    01199     //
    01200
    01201     if (Irp->Type != IO_TYPE_IRP) {
    01202
    01203         ASSERT(Irp->Type == IO_TYPE_IRP);
    01204         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    01205                      IO_CALL_DRIVER_IRP_TYPE_INVALID,
    01206                      (ULONG_PTR)Irp,
    01207                      0,
    01208                      0);
    01209     }
    01210
    01211     if (!IovpValidateDeviceObject(DeviceObject)) {
    01212         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    01213                      IO_CALL_DRIVER_INVALID_DEVICE_OBJECT,
    01214                      (ULONG_PTR)DeviceObject,
    01215                      0,
    01216                      0);
    01217     }
    01218
    01219     SPECIALIRP_IOF_CALL_1(&Irp, DeviceObject, &iofCallDriverStackData) ;
    01220
    01221     //
    01222     // Update the IRP stack to point to the next location.
    01223     //
    01224     Irp->CurrentLocation--;
    01225
    01226     if (Irp->CurrentLocation <= 0) {
    01227         KeBugCheckEx( NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR) Irp, 0, 0, 0 );
    01228     }
    01229
    01230     irpSp = IoGetNextIrpStackLocation( Irp );
    01231     Irp->Tail.Overlay.CurrentStackLocation = irpSp;
    01232
    01233     //
    01234     // Save a pointer to the device object for this request so that it can
    01235     // be used later in completion.
    01236     //
    01237
    01238     irpSp->DeviceObject = DeviceObject;
    01239
    01240     //
    01241     // Invoke the driver at its dispatch routine entry point.
    01242     //
    01243
    01244     driverObject = DeviceObject->DriverObject;
    01245
    01246     dispatchRoutine = driverObject->MajorFunction[irpSp->MajorFunction] ;
    01247
    01248     saveIrql = KeGetCurrentIrql();
    01249
    01250     PERFINFO_DRIVER_MAJORFUNCTION_CALL(Irp, irpSp, driverObject);
    01251
    01252     status = dispatchRoutine( DeviceObject, Irp );
    01253
    01254     PERFINFO_DRIVER_MAJORFUNCTION_RETURN(Irp, irpSp, driverObject);
    01255
    01256     if (saveIrql != KeGetCurrentIrql()) {
    01257
    01258         DbgPrint( "IO: IoCallDriver( Driver object: %p  Device object: %p  Irp: %p )\n",
    01259                   driverObject,
    01260                   DeviceObject,
    01261                   Irp
    01262                 );
    01263         DbgPrint( "    Irql before: %x  != After: %x\n", saveIrql, KeGetCurrentIrql() );
    01264
    01265         ASSERT(saveIrql == KeGetCurrentIrql());
    01266
    01267         KeBugCheckEx(DRIVER_VERIFIER_IOMANAGER_VIOLATION,
    01268                      IO_CALL_DRIVER_IRQL_NOT_EQUAL,
    01269                      saveIrql,
    01270                      KeGetCurrentIrql(),
    01271                      0);
    01272     }
    01273
    01274     SPECIALIRP_IOF_CALL_2(Irp, DeviceObject, dispatchRoutine, &status, &iofCallDriverStackData) ;
    01275
    01276     return status;
    01277 }
    01278
    01279 VOID
    01280 IovInitializeIrp(
    01281     PIRP    Irp,
    01282     USHORT  PacketSize,
    01283     CCHAR   StackSize
    01284     )
    01285 {
    01286     BOOLEAN initializeHandled ;
    01287
    01288     if (IovpVerifierLevel < 2) {
    01289         return;
    01290     }
    01291
    01292     SPECIALIRP_IO_INITIALIZE_IRP(Irp, PacketSize, StackSize, &initializeHandled) ;
    01293
    01294 }
    01295
    01296 VOID
    01297 IovAttachDeviceToDeviceStack(
    01298     PDEVICE_OBJECT  SourceDevice,
    01299     PDEVICE_OBJECT  TargetDevice
    01300     )
    01301 {
    01302     if (IovpVerifierLevel < 2) {
    01303         return;
    01304     }
    01305
    01306     SPECIALIRP_IO_ATTACH_DEVICE_TO_DEVICE_STACK(SourceDevice, TargetDevice);
    01307 }
    01308
    01309 VOID
    01310 IovDeleteDevice(
    01311     PDEVICE_OBJECT  DeleteDevice
    01312     )
    01313 {
    01314     if (IovpVerifierLevel < 2) {
    01315         return;
    01316     }
    01317
    01318     SPECIALIRP_IO_DELETE_DEVICE(DeleteDevice);
    01319 }
    01320
    01321 VOID
    01322 IovDetachDevice(
    01323     PDEVICE_OBJECT  TargetDevice
    01324     )
    01325 {
    01326     if (IovpVerifierLevel < 2) {
    01327         return;
    01328     }
    01329     SPECIALIRP_IO_DETACH_DEVICE(TargetDevice);
    01330 }
    01331
    01332 BOOLEAN
    01333 IovCancelIrp(
    01334     PIRP    Irp,
    01335     BOOLEAN *returnValue
    01336     )
    01337 {
    01338 #ifndef NO_SPECIAL_IRP
    01339     BOOLEAN cancelHandled ;
    01340
    01341     SPECIALIRP_IO_CANCEL_IRP(Irp, &cancelHandled, returnValue) ;
    01342
    01343     if (cancelHandled) {
    01344
    01345        return TRUE ;
    01346     }
    01347 #endif
    01348     return FALSE;
    01349 }