VirtualBox

source: vbox/trunk/src/VBox/Runtime/r0drv/nt/mp-r0drv-nt.cpp@ 92871

最後變更 在這個檔案從92871是 92871,由 vboxsync 提交於 3 年 前

IPRT/rtMpNtSetTargetProcessorDpc: Use AssertLogRel so we can hopefully track down the RTTimerCreatEx/specific issue.
  • 屬性 svn:eol-style 設為 native
  • 屬性 svn:keywords 設為 Author Date Id Revision
檔案大小: 76.6 KB
行 
1/* $Id: mp-r0drv-nt.cpp 92871 2021-12-11 00:23:15Z vboxsync $ */
2/** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, NT.
4 */
5
6/*
7 * Copyright (C) 2008-2020 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.alldomusa.eu.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include "the-nt-kernel.h"
32
33#include <iprt/mp.h>
34#include <iprt/cpuset.h>
35#include <iprt/err.h>
36#include <iprt/asm.h>
37#include <iprt/log.h>
38#include <iprt/mem.h>
39#include <iprt/time.h>
40#include "r0drv/mp-r0drv.h"
41#include "symdb.h"
42#include "internal-r0drv-nt.h"
43#include "internal/mp.h"
44
45
46/*********************************************************************************************************************************
47* Structures and Typedefs *
48*********************************************************************************************************************************/
49typedef enum
50{
51 RT_NT_CPUID_SPECIFIC,
52 RT_NT_CPUID_PAIR,
53 RT_NT_CPUID_OTHERS,
54 RT_NT_CPUID_ALL
55} RT_NT_CPUID;
56
57
58/**
59 * Used by the RTMpOnSpecific.
60 */
61typedef struct RTMPNTONSPECIFICARGS
62{
63 /** Set if we're executing. */
64 bool volatile fExecuting;
65 /** Set when done executing. */
66 bool volatile fDone;
67 /** Number of references to this heap block. */
68 uint32_t volatile cRefs;
69 /** Event that the calling thread is waiting on. */
70 KEVENT DoneEvt;
71 /** The deferred procedure call object. */
72 KDPC Dpc;
73 /** The callback argument package. */
74 RTMPARGS CallbackArgs;
75} RTMPNTONSPECIFICARGS;
76/** Pointer to an argument/state structure for RTMpOnSpecific on NT. */
77typedef RTMPNTONSPECIFICARGS *PRTMPNTONSPECIFICARGS;
78
79
80/*********************************************************************************************************************************
81* Defined Constants And Macros *
82*********************************************************************************************************************************/
83/** Inactive bit for g_aidRtMpNtByCpuSetIdx. */
84#define RTMPNT_ID_F_INACTIVE RT_BIT_32(31)
85
86
87/*********************************************************************************************************************************
88* Global Variables *
89*********************************************************************************************************************************/
90/** Maximum number of processor groups. */
91uint32_t g_cRtMpNtMaxGroups;
92/** Maximum number of processors. */
93uint32_t g_cRtMpNtMaxCpus;
94/** Number of active processors. */
95uint32_t volatile g_cRtMpNtActiveCpus;
96/** The NT CPU set.
97 * KeQueryActiveProcssors() cannot be called at all IRQLs and therefore we'll
98 * have to cache it. Fortunately, NT doesn't really support taking CPUs offline,
99 * and taking them online was introduced with W2K8 where it is intended for virtual
100 * machines and not real HW. We update this, g_cRtMpNtActiveCpus and
101 * g_aidRtMpNtByCpuSetIdx from the rtR0NtMpProcessorChangeCallback.
102 */
103RTCPUSET g_rtMpNtCpuSet;
104
105/** Static per group info.
106 * @remarks With 256 groups this takes up 33KB. */
107static struct
108{
109 /** The max CPUs in the group. */
110 uint16_t cMaxCpus;
111 /** The number of active CPUs at the time of initialization. */
112 uint16_t cActiveCpus;
113 /** CPU set indexes for each CPU in the group. */
114 int16_t aidxCpuSetMembers[64];
115} g_aRtMpNtCpuGroups[256];
116/** Maps CPU set indexes to RTCPUID.
117 * Inactive CPUs has bit 31 set (RTMPNT_ID_F_INACTIVE) so we can identify them
118 * and shuffle duplicates during CPU hotplugging. We assign temporary IDs to
119 * the inactive CPUs starting at g_cRtMpNtMaxCpus - 1, ASSUMING that active
120 * CPUs has IDs from 0 to g_cRtMpNtActiveCpus. */
121RTCPUID g_aidRtMpNtByCpuSetIdx[RTCPUSET_MAX_CPUS];
122/** The handle of the rtR0NtMpProcessorChangeCallback registration. */
123static PVOID g_pvMpCpuChangeCallback = NULL;
124/** Size of the KAFFINITY_EX structure.
125 * This increased from 20 to 32 bitmap words in the 2020 H2 windows 10 release
126 * (i.e. 1280 to 2048 CPUs). We expect this to increase in the future. */
127static size_t g_cbRtMpNtKaffinityEx = RT_UOFFSETOF(KAFFINITY_EX, Bitmap)
128 + RT_SIZEOFMEMB(KAFFINITY_EX, Bitmap[0]) * 256;
129/** The size value of the KAFFINITY_EX structure. */
130static uint16_t g_cRtMpNtKaffinityExEntries = 256;
131
132
133/*********************************************************************************************************************************
134* Internal Functions *
135*********************************************************************************************************************************/
136static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
137 PNTSTATUS prcOperationStatus);
138static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo);
139
140
141
142/**
143 * Initalizes multiprocessor globals (called by rtR0InitNative).
144 *
145 * @returns IPRT status code.
146 * @param pOsVerInfo Version information.
147 */
148DECLHIDDEN(int) rtR0MpNtInit(RTNTSDBOSVER const *pOsVerInfo)
149{
150#define MY_CHECK_BREAK(a_Check, a_DbgPrintArgs) \
151 AssertMsgBreakStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
152#define MY_CHECK_RETURN(a_Check, a_DbgPrintArgs, a_rcRet) \
153 AssertMsgReturnStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs, a_rcRet)
154#define MY_CHECK(a_Check, a_DbgPrintArgs) \
155 AssertMsgStmt(a_Check, a_DbgPrintArgs, DbgPrint a_DbgPrintArgs; rc = VERR_INTERNAL_ERROR_4 )
156
157 /*
158 * API combination checks.
159 */
160 MY_CHECK_RETURN(!g_pfnrtKeSetTargetProcessorDpcEx || g_pfnrtKeGetProcessorNumberFromIndex,
161 ("IPRT: Fatal: Missing KeSetTargetProcessorDpcEx without KeGetProcessorNumberFromIndex!\n"),
162 VERR_SYMBOL_NOT_FOUND);
163
164 /*
165 * Get max number of processor groups.
166 *
167 * We may need to upadjust this number below, because windows likes to keep
168 * all options open when it comes to hotplugged CPU group assignments. A
169 * server advertising up to 64 CPUs in the ACPI table will get a result of
170 * 64 from KeQueryMaximumGroupCount. That makes sense. However, when windows
171 * server 2012 does a two processor group setup for it, the sum of the
172 * GroupInfo[*].MaximumProcessorCount members below is 128. This is probably
173 * because windows doesn't want to make decisions grouping of hotpluggable CPUs.
174 * So, we need to bump the maximum count to 128 below do deal with this as we
175 * want to have valid CPU set indexes for all potential CPUs - how could we
176 * otherwise use the RTMpGetSet() result and also RTCpuSetCount(RTMpGetSet())
177 * should equal RTMpGetCount().
178 */
179 if (g_pfnrtKeQueryMaximumGroupCount)
180 {
181 g_cRtMpNtMaxGroups = g_pfnrtKeQueryMaximumGroupCount();
182 MY_CHECK_RETURN(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
183 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
184 VERR_MP_TOO_MANY_CPUS);
185 }
186 else
187 g_cRtMpNtMaxGroups = 1;
188
189 /*
190 * Get max number CPUs.
191 * This also defines the range of NT CPU indexes, RTCPUID and index into RTCPUSET.
192 */
193 if (g_pfnrtKeQueryMaximumProcessorCountEx)
194 {
195 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
196 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
197 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCountEx]\n",
198 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
199 VERR_MP_TOO_MANY_CPUS);
200 }
201 else if (g_pfnrtKeQueryMaximumProcessorCount)
202 {
203 g_cRtMpNtMaxCpus = g_pfnrtKeQueryMaximumProcessorCount();
204 MY_CHECK_RETURN(g_cRtMpNtMaxCpus <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxCpus > 0,
205 ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, max %u [KeQueryMaximumProcessorCount]\n",
206 g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS),
207 VERR_MP_TOO_MANY_CPUS);
208 }
209 else if (g_pfnrtKeQueryActiveProcessors)
210 {
211 KAFFINITY fActiveProcessors = g_pfnrtKeQueryActiveProcessors();
212 MY_CHECK_RETURN(fActiveProcessors != 0,
213 ("IPRT: Fatal: KeQueryActiveProcessors returned 0!\n"),
214 VERR_INTERNAL_ERROR_2);
215 g_cRtMpNtMaxCpus = 0;
216 do
217 {
218 g_cRtMpNtMaxCpus++;
219 fActiveProcessors >>= 1;
220 } while (fActiveProcessors);
221 }
222 else
223 g_cRtMpNtMaxCpus = KeNumberProcessors;
224
225 /*
226 * Just because we're a bit paranoid about getting something wrong wrt to the
227 * kernel interfaces, we try 16 times to get the KeQueryActiveProcessorCountEx
228 * and KeQueryLogicalProcessorRelationship information to match up.
229 */
230 for (unsigned cTries = 0;; cTries++)
231 {
232 /*
233 * Get number of active CPUs.
234 */
235 if (g_pfnrtKeQueryActiveProcessorCountEx)
236 {
237 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
238 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
239 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCountEx]\n",
240 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
241 VERR_MP_TOO_MANY_CPUS);
242 }
243 else if (g_pfnrtKeQueryActiveProcessorCount)
244 {
245 g_cRtMpNtActiveCpus = g_pfnrtKeQueryActiveProcessorCount(NULL);
246 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus && g_cRtMpNtActiveCpus > 0,
247 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u [KeQueryActiveProcessorCount]\n",
248 g_cRtMpNtMaxGroups, g_cRtMpNtMaxCpus),
249 VERR_MP_TOO_MANY_CPUS);
250 }
251 else
252 g_cRtMpNtActiveCpus = g_cRtMpNtMaxCpus;
253
254 /*
255 * Query the details for the groups to figure out which CPUs are online as
256 * well as the NT index limit.
257 */
258 for (unsigned i = 0; i < RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx); i++)
259#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
260 g_aidRtMpNtByCpuSetIdx[i] = NIL_RTCPUID;
261#else
262 g_aidRtMpNtByCpuSetIdx[i] = i < g_cRtMpNtMaxCpus ? i : NIL_RTCPUID;
263#endif
264 for (unsigned idxGroup = 0; idxGroup < RT_ELEMENTS(g_aRtMpNtCpuGroups); idxGroup++)
265 {
266 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = 0;
267 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
268 for (unsigned idxMember = 0; idxMember < RT_ELEMENTS(g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers); idxMember++)
269 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = -1;
270 }
271
272 if (g_pfnrtKeQueryLogicalProcessorRelationship)
273 {
274 MY_CHECK_RETURN(g_pfnrtKeGetProcessorIndexFromNumber,
275 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
276 VERR_SYMBOL_NOT_FOUND);
277 MY_CHECK_RETURN(g_pfnrtKeGetProcessorNumberFromIndex,
278 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeGetProcessorIndexFromNumber!\n"),
279 VERR_SYMBOL_NOT_FOUND);
280 MY_CHECK_RETURN(g_pfnrtKeSetTargetProcessorDpcEx,
281 ("IPRT: Fatal: Found KeQueryLogicalProcessorRelationship but not KeSetTargetProcessorDpcEx!\n"),
282 VERR_SYMBOL_NOT_FOUND);
283
284 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = NULL;
285 int rc = rtR0NtInitQueryGroupRelations(&pInfo);
286 if (RT_FAILURE(rc))
287 return rc;
288
289 MY_CHECK(pInfo->Group.MaximumGroupCount == g_cRtMpNtMaxGroups,
290 ("IPRT: Fatal: MaximumGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
291 pInfo->Group.MaximumGroupCount, g_cRtMpNtMaxGroups));
292 MY_CHECK(pInfo->Group.ActiveGroupCount > 0 && pInfo->Group.ActiveGroupCount <= g_cRtMpNtMaxGroups,
293 ("IPRT: Fatal: ActiveGroupCount=%u != g_cRtMpNtMaxGroups=%u!\n",
294 pInfo->Group.ActiveGroupCount, g_cRtMpNtMaxGroups));
295
296 /*
297 * First we need to recalc g_cRtMpNtMaxCpus (see above).
298 */
299 uint32_t cMaxCpus = 0;
300 uint32_t idxGroup;
301 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
302 {
303 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
304 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
305 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
306 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
307 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
308 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
309 cMaxCpus += pGrpInfo->MaximumProcessorCount;
310 }
311 if (cMaxCpus > g_cRtMpNtMaxCpus && RT_SUCCESS(rc))
312 {
313 DbgPrint("IPRT: g_cRtMpNtMaxCpus=%u -> %u\n", g_cRtMpNtMaxCpus, cMaxCpus);
314#ifndef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
315 uint32_t i = RT_MIN(cMaxCpus, RT_ELEMENTS(g_aidRtMpNtByCpuSetIdx));
316 while (i-- > g_cRtMpNtMaxCpus)
317 g_aidRtMpNtByCpuSetIdx[i] = i;
318#endif
319 g_cRtMpNtMaxCpus = cMaxCpus;
320 if (g_cRtMpNtMaxGroups > RTCPUSET_MAX_CPUS)
321 {
322 MY_CHECK(g_cRtMpNtMaxGroups <= RTCPUSET_MAX_CPUS && g_cRtMpNtMaxGroups > 0,
323 ("IPRT: Fatal: g_cRtMpNtMaxGroups=%u, max %u\n", g_cRtMpNtMaxGroups, RTCPUSET_MAX_CPUS));
324 rc = VERR_MP_TOO_MANY_CPUS;
325 }
326 }
327
328 /*
329 * Calc online mask, partition IDs and such.
330 *
331 * Also check ASSUMPTIONS:
332 *
333 * 1. Processor indexes going from 0 and up to
334 * KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
335 *
336 * 2. Currently valid processor indexes, i.e. accepted by
337 * KeGetProcessorIndexFromNumber & KeGetProcessorNumberFromIndex, goes
338 * from 0 thru KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS) - 1.
339 *
340 * 3. PROCESSOR_GROUP_INFO::MaximumProcessorCount gives the number of
341 * relevant bits in the ActiveProcessorMask (from LSB).
342 *
343 * 4. Active processor count found in KeQueryLogicalProcessorRelationship
344 * output matches what KeQueryActiveProcessorCountEx(ALL) returns.
345 *
346 * 5. Active + inactive processor counts in same does not exceed
347 * KeQueryMaximumProcessorCountEx(ALL).
348 *
349 * Note! Processor indexes are assigned as CPUs come online and are not
350 * preallocated according to group maximums. Since CPUS are only taken
351 * online and never offlined, this means that internal CPU bitmaps are
352 * never sparse and no time is wasted scanning unused bits.
353 *
354 * Unfortunately, it means that ring-3 cannot easily guess the index
355 * assignments when hotswapping is used, and must use GIP when available.
356 */
357 RTCpuSetEmpty(&g_rtMpNtCpuSet);
358 uint32_t cInactive = 0;
359 uint32_t cActive = 0;
360 uint32_t idxCpuMax = 0;
361 uint32_t idxCpuSetNextInactive = g_cRtMpNtMaxCpus - 1;
362 for (idxGroup = 0; RT_SUCCESS(rc) && idxGroup < pInfo->Group.ActiveGroupCount; idxGroup++)
363 {
364 const PROCESSOR_GROUP_INFO *pGrpInfo = &pInfo->Group.GroupInfo[idxGroup];
365 MY_CHECK_BREAK(pGrpInfo->MaximumProcessorCount <= MAXIMUM_PROC_PER_GROUP,
366 ("IPRT: Fatal: MaximumProcessorCount=%u\n", pGrpInfo->MaximumProcessorCount));
367 MY_CHECK_BREAK(pGrpInfo->ActiveProcessorCount <= pGrpInfo->MaximumProcessorCount,
368 ("IPRT: Fatal: ActiveProcessorCount=%u > MaximumProcessorCount=%u\n",
369 pGrpInfo->ActiveProcessorCount, pGrpInfo->MaximumProcessorCount));
370
371 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = pGrpInfo->MaximumProcessorCount;
372 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = pGrpInfo->ActiveProcessorCount;
373
374 for (uint32_t idxMember = 0; idxMember < pGrpInfo->MaximumProcessorCount; idxMember++)
375 {
376 PROCESSOR_NUMBER ProcNum;
377 ProcNum.Group = (USHORT)idxGroup;
378 ProcNum.Number = (UCHAR)idxMember;
379 ProcNum.Reserved = 0;
380 ULONG idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
381 if (idxCpu != INVALID_PROCESSOR_INDEX)
382 {
383 MY_CHECK_BREAK(idxCpu < g_cRtMpNtMaxCpus && idxCpu < RTCPUSET_MAX_CPUS, /* ASSUMPTION #1 */
384 ("IPRT: Fatal: idxCpu=%u >= g_cRtMpNtMaxCpus=%u (RTCPUSET_MAX_CPUS=%u)\n",
385 idxCpu, g_cRtMpNtMaxCpus, RTCPUSET_MAX_CPUS));
386 if (idxCpu > idxCpuMax)
387 idxCpuMax = idxCpu;
388 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpu;
389#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
390 g_aidRtMpNtByCpuSetIdx[idxCpu] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember);
391#endif
392
393 ProcNum.Group = UINT16_MAX;
394 ProcNum.Number = UINT8_MAX;
395 ProcNum.Reserved = UINT8_MAX;
396 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(idxCpu, &ProcNum);
397 MY_CHECK_BREAK(NT_SUCCESS(rcNt),
398 ("IPRT: Fatal: KeGetProcessorNumberFromIndex(%u,) -> %#x!\n", idxCpu, rcNt));
399 MY_CHECK_BREAK(ProcNum.Group == idxGroup && ProcNum.Number == idxMember,
400 ("IPRT: Fatal: KeGetProcessorXxxxFromYyyy roundtrip error for %#x! Group: %u vs %u, Number: %u vs %u\n",
401 idxCpu, ProcNum.Group, idxGroup, ProcNum.Number, idxMember));
402
403 if (pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember))
404 {
405 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, idxCpu);
406 cActive++;
407 }
408 else
409 cInactive++; /* (This is a little unexpected, but not important as long as things add up below.) */
410 }
411 else
412 {
413 /* Must be not present / inactive when KeGetProcessorIndexFromNumber fails. */
414 MY_CHECK_BREAK(!(pGrpInfo->ActiveProcessorMask & RT_BIT_64(idxMember)),
415 ("IPRT: Fatal: KeGetProcessorIndexFromNumber(%u/%u) failed but CPU is active! cMax=%u cActive=%u fActive=%p\n",
416 idxGroup, idxMember, pGrpInfo->MaximumProcessorCount, pGrpInfo->ActiveProcessorCount,
417 pGrpInfo->ActiveProcessorMask));
418 cInactive++;
419 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
420 {
421 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
422#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
423 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
424 | RTMPNT_ID_F_INACTIVE;
425#endif
426 idxCpuSetNextInactive--;
427 }
428 }
429 }
430 }
431
432 MY_CHECK(cInactive + cActive <= g_cRtMpNtMaxCpus, /* ASSUMPTION #5 (not '==' because of inactive groups) */
433 ("IPRT: Fatal: cInactive=%u + cActive=%u > g_cRtMpNtMaxCpus=%u\n", cInactive, cActive, g_cRtMpNtMaxCpus));
434
435 /* Deal with inactive groups using KeQueryMaximumProcessorCountEx or as
436 best as we can by as best we can by stipulating maximum member counts
437 from the previous group. */
438 if ( RT_SUCCESS(rc)
439 && idxGroup < pInfo->Group.MaximumGroupCount)
440 {
441 uint16_t cInactiveLeft = g_cRtMpNtMaxCpus - (cInactive + cActive);
442 while (idxGroup < pInfo->Group.MaximumGroupCount)
443 {
444 uint32_t cMaxMembers = 0;
445 if (g_pfnrtKeQueryMaximumProcessorCountEx)
446 cMaxMembers = g_pfnrtKeQueryMaximumProcessorCountEx(idxGroup);
447 if (cMaxMembers != 0 || cInactiveLeft == 0)
448 AssertStmt(cMaxMembers <= cInactiveLeft, cMaxMembers = cInactiveLeft);
449 else
450 {
451 uint16_t cGroupsLeft = pInfo->Group.MaximumGroupCount - idxGroup;
452 cMaxMembers = pInfo->Group.GroupInfo[idxGroup - 1].MaximumProcessorCount;
453 while (cMaxMembers * cGroupsLeft < cInactiveLeft)
454 cMaxMembers++;
455 if (cMaxMembers > cInactiveLeft)
456 cMaxMembers = cInactiveLeft;
457 }
458
459 g_aRtMpNtCpuGroups[idxGroup].cMaxCpus = (uint16_t)cMaxMembers;
460 g_aRtMpNtCpuGroups[idxGroup].cActiveCpus = 0;
461 for (uint16_t idxMember = 0; idxMember < cMaxMembers; idxMember++)
462 if (idxCpuSetNextInactive >= g_cRtMpNtActiveCpus)
463 {
464 g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember] = idxCpuSetNextInactive;
465#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
466 g_aidRtMpNtByCpuSetIdx[idxCpuSetNextInactive] = RTMPCPUID_FROM_GROUP_AND_NUMBER(idxGroup, idxMember)
467 | RTMPNT_ID_F_INACTIVE;
468#endif
469 idxCpuSetNextInactive--;
470 }
471 cInactiveLeft -= cMaxMembers;
472 idxGroup++;
473 }
474 }
475
476 /* We're done with pInfo now, free it so we can start returning when assertions fail. */
477 RTMemFree(pInfo);
478 if (RT_FAILURE(rc)) /* MY_CHECK_BREAK sets rc. */
479 return rc;
480 MY_CHECK_RETURN(cActive >= g_cRtMpNtActiveCpus,
481 ("IPRT: Fatal: cActive=%u < g_cRtMpNtActiveCpus=%u - CPUs removed?\n", cActive, g_cRtMpNtActiveCpus),
482 VERR_INTERNAL_ERROR_3);
483 MY_CHECK_RETURN(idxCpuMax < cActive, /* ASSUMPTION #2 */
484 ("IPRT: Fatal: idCpuMax=%u >= cActive=%u! Unexpected CPU index allocation. CPUs removed?\n",
485 idxCpuMax, cActive),
486 VERR_INTERNAL_ERROR_4);
487
488 /* Retry if CPUs were added. */
489 if ( cActive != g_cRtMpNtActiveCpus
490 && cTries < 16)
491 continue;
492 MY_CHECK_RETURN(cActive == g_cRtMpNtActiveCpus, /* ASSUMPTION #4 */
493 ("IPRT: Fatal: cActive=%u != g_cRtMpNtActiveCpus=%u\n", cActive, g_cRtMpNtActiveCpus),
494 VERR_INTERNAL_ERROR_5);
495 }
496 else
497 {
498 /* Legacy: */
499 MY_CHECK_RETURN(g_cRtMpNtMaxGroups == 1, ("IPRT: Fatal: Missing KeQueryLogicalProcessorRelationship!\n"),
500 VERR_SYMBOL_NOT_FOUND);
501
502 /** @todo Is it possible that the affinity mask returned by
503 * KeQueryActiveProcessors is sparse? */
504 if (g_pfnrtKeQueryActiveProcessors)
505 RTCpuSetFromU64(&g_rtMpNtCpuSet, g_pfnrtKeQueryActiveProcessors());
506 else if (g_cRtMpNtMaxCpus < 64)
507 RTCpuSetFromU64(&g_rtMpNtCpuSet, (UINT64_C(1) << g_cRtMpNtMaxCpus) - 1);
508 else
509 {
510 MY_CHECK_RETURN(g_cRtMpNtMaxCpus == 64, ("IPRT: Fatal: g_cRtMpNtMaxCpus=%u, expect 64 or less\n", g_cRtMpNtMaxCpus),
511 VERR_MP_TOO_MANY_CPUS);
512 RTCpuSetFromU64(&g_rtMpNtCpuSet, UINT64_MAX);
513 }
514
515 g_aRtMpNtCpuGroups[0].cMaxCpus = g_cRtMpNtMaxCpus;
516 g_aRtMpNtCpuGroups[0].cActiveCpus = g_cRtMpNtMaxCpus;
517 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
518 {
519 g_aRtMpNtCpuGroups[0].aidxCpuSetMembers[i] = i;
520#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
521 g_aidRtMpNtByCpuSetIdx[i] = RTMPCPUID_FROM_GROUP_AND_NUMBER(0, i);
522#endif
523 }
524 }
525
526 /*
527 * Register CPU hot plugging callback (it also counts active CPUs).
528 */
529 Assert(g_pvMpCpuChangeCallback == NULL);
530 if (g_pfnrtKeRegisterProcessorChangeCallback)
531 {
532 MY_CHECK_RETURN(g_pfnrtKeDeregisterProcessorChangeCallback,
533 ("IPRT: Fatal: KeRegisterProcessorChangeCallback without KeDeregisterProcessorChangeCallback!\n"),
534 VERR_SYMBOL_NOT_FOUND);
535
536 RTCPUSET const ActiveSetCopy = g_rtMpNtCpuSet;
537 RTCpuSetEmpty(&g_rtMpNtCpuSet);
538 uint32_t const cActiveCpus = g_cRtMpNtActiveCpus;
539 g_cRtMpNtActiveCpus = 0;
540
541 g_pvMpCpuChangeCallback = g_pfnrtKeRegisterProcessorChangeCallback(rtR0NtMpProcessorChangeCallback, NULL /*pvUser*/,
542 KE_PROCESSOR_CHANGE_ADD_EXISTING);
543 if (g_pvMpCpuChangeCallback)
544 {
545 if (cActiveCpus == g_cRtMpNtActiveCpus)
546 { /* likely */ }
547 else
548 {
549 g_pfnrtKeDeregisterProcessorChangeCallback(g_pvMpCpuChangeCallback);
550 if (cTries < 16)
551 {
552 /* Retry if CPUs were added. */
553 MY_CHECK_RETURN(g_cRtMpNtActiveCpus >= cActiveCpus,
554 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u < cActiveCpus=%u! CPUs removed?\n",
555 g_cRtMpNtActiveCpus, cActiveCpus),
556 VERR_INTERNAL_ERROR_2);
557 MY_CHECK_RETURN(g_cRtMpNtActiveCpus <= g_cRtMpNtMaxCpus,
558 ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u > g_cRtMpNtMaxCpus=%u!\n",
559 g_cRtMpNtActiveCpus, g_cRtMpNtMaxCpus),
560 VERR_INTERNAL_ERROR_2);
561 continue;
562 }
563 MY_CHECK_RETURN(0, ("IPRT: Fatal: g_cRtMpNtActiveCpus=%u cActiveCpus=%u\n", g_cRtMpNtActiveCpus, cActiveCpus),
564 VERR_INTERNAL_ERROR_3);
565 }
566 }
567 else
568 {
569 AssertFailed();
570 g_rtMpNtCpuSet = ActiveSetCopy;
571 g_cRtMpNtActiveCpus = cActiveCpus;
572 }
573 }
574 break;
575 } /* Retry loop for stable active CPU count. */
576
577#undef MY_CHECK_RETURN
578
579 /*
580 * Special IPI fun for RTMpPokeCpu.
581 *
582 * On Vista and later the DPC method doesn't seem to reliably send IPIs,
583 * so we have to use alternative methods.
584 *
585 * On AMD64 We used to use the HalSendSoftwareInterrupt API (also x86 on
586 * W10+), it looks faster and more convenient to use, however we're either
587 * using it wrong or it doesn't reliably do what we want (see @bugref{8343}).
588 *
589 * The HalRequestIpip API is thus far the only alternative to KeInsertQueueDpc
590 * for doing targetted IPIs. Trouble with this API is that it changed
591 * fundamentally in Window 7 when they added support for lots of processors.
592 *
593 * If we really think we cannot use KeInsertQueueDpc, we use the broadcast IPI
594 * API KeIpiGenericCall.
595 */
596 if ( pOsVerInfo->uMajorVer > 6
597 || (pOsVerInfo->uMajorVer == 6 && pOsVerInfo->uMinorVer > 0))
598 g_pfnrtHalRequestIpiPreW7 = NULL;
599 else
600 g_pfnrtHalRequestIpiW7Plus = NULL;
601
602 if ( g_pfnrtHalRequestIpiW7Plus
603 && g_pfnrtKeInitializeAffinityEx
604 && g_pfnrtKeAddProcessorAffinityEx
605 && g_pfnrtKeGetProcessorIndexFromNumber)
606 {
607 /* Determine the real size of the KAFFINITY_EX structure. */
608 size_t const cbAffinity = _8K;
609 PKAFFINITY_EX pAffinity = (PKAFFINITY_EX)RTMemAllocZ(cbAffinity);
610 AssertReturn(pAffinity, VERR_NO_MEMORY);
611 size_t const cMaxEntries = (cbAffinity - RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0])) / sizeof(pAffinity->Bitmap[0]);
612 g_pfnrtKeInitializeAffinityEx(pAffinity);
613 if (pAffinity->Size > 1 && pAffinity->Size <= cMaxEntries)
614 {
615 g_cRtMpNtKaffinityExEntries = pAffinity->Size;
616 g_cbRtMpNtKaffinityEx = pAffinity->Size * sizeof(pAffinity->Bitmap[0]) + RT_UOFFSETOF(KAFFINITY_EX, Bitmap[0]);
617 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingHalRequestIpiW7Plus;
618 RTMemFree(pAffinity);
619 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingHalRequestIpiW7Plus\n");
620 return VINF_SUCCESS;
621 }
622 DbgPrint("IPRT: RTMpPoke can't use rtMpPokeCpuUsingHalRequestIpiW7Plus! pAffinity->Size=%u\n", pAffinity->Size);
623 AssertReleaseMsg(pAffinity->Size <= cMaxEntries, ("%#x\n", pAffinity->Size)); /* stack is toast if larger (32768 CPUs). */
624 RTMemFree(pAffinity);
625 }
626
627 if (pOsVerInfo->uMajorVer >= 6 && g_pfnrtKeIpiGenericCall)
628 {
629 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingBroadcastIpi\n");
630 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingBroadcastIpi;
631 }
632 else if (g_pfnrtKeSetTargetProcessorDpc)
633 {
634 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingDpc\n");
635 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingDpc;
636 /* Windows XP should send always send an IPI -> VERIFY */
637 }
638 else
639 {
640 DbgPrint("IPRT: RTMpPoke => rtMpPokeCpuUsingFailureNotSupported\n");
641 Assert(pOsVerInfo->uMajorVer == 3 && pOsVerInfo->uMinorVer <= 50);
642 g_pfnrtMpPokeCpuWorker = rtMpPokeCpuUsingFailureNotSupported;
643 }
644
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Called by rtR0TermNative.
651 */
652DECLHIDDEN(void) rtR0MpNtTerm(void)
653{
654 /*
655 * Deregister the processor change callback.
656 */
657 PVOID pvMpCpuChangeCallback = g_pvMpCpuChangeCallback;
658 g_pvMpCpuChangeCallback = NULL;
659 if (pvMpCpuChangeCallback)
660 {
661 AssertReturnVoid(g_pfnrtKeDeregisterProcessorChangeCallback);
662 g_pfnrtKeDeregisterProcessorChangeCallback(pvMpCpuChangeCallback);
663 }
664}
665
666
667DECLHIDDEN(int) rtR0MpNotificationNativeInit(void)
668{
669 return VINF_SUCCESS;
670}
671
672
673DECLHIDDEN(void) rtR0MpNotificationNativeTerm(void)
674{
675}
676
677
678/**
679 * Implements the NT PROCESSOR_CALLBACK_FUNCTION callback function.
680 *
681 * This maintains the g_rtMpNtCpuSet and works MP notification callbacks. When
682 * registered, it's called for each active CPU in the system, avoiding racing
683 * CPU hotplugging (as well as testing the callback).
684 *
685 * @param pvUser User context (not used).
686 * @param pChangeCtx Change context (in).
687 * @param prcOperationStatus Operation status (in/out).
688 *
689 * @remarks ASSUMES no concurrent execution of KeProcessorAddCompleteNotify
690 * notification callbacks. At least during callback registration
691 * callout, we're owning KiDynamicProcessorLock.
692 *
693 * @remarks When registering the handler, we first get KeProcessorAddStartNotify
694 * callbacks for all active CPUs, and after they all succeed we get the
695 * KeProcessorAddCompleteNotify callbacks.
696 */
697static VOID __stdcall rtR0NtMpProcessorChangeCallback(void *pvUser, PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT pChangeCtx,
698 PNTSTATUS prcOperationStatus)
699{
700 RT_NOREF(pvUser, prcOperationStatus);
701 switch (pChangeCtx->State)
702 {
703 /*
704 * Check whether we can deal with the CPU, failing the start operation if we
705 * can't. The checks we are doing here are to avoid complicated/impossible
706 * cases in KeProcessorAddCompleteNotify. They are really just verify specs.
707 */
708 case KeProcessorAddStartNotify:
709 {
710 NTSTATUS rcNt = STATUS_SUCCESS;
711 if (pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS)
712 {
713 if (pChangeCtx->NtNumber >= g_cRtMpNtMaxCpus)
714 {
715 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is higher than the max CPU count (%u)!\n",
716 pChangeCtx->NtNumber, g_cRtMpNtMaxCpus);
717 rcNt = STATUS_INTERNAL_ERROR;
718 }
719
720 /* The ProcessNumber field was introduced in Windows 7. */
721 PROCESSOR_NUMBER ProcNum;
722 if (g_pfnrtKeGetProcessorIndexFromNumber)
723 {
724 ProcNum = pChangeCtx->ProcNumber;
725 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
726 if (idxCpu != pChangeCtx->NtNumber)
727 {
728 DbgPrint("IPRT: KeProcessorAddStartNotify failure: g_pfnrtKeGetProcessorIndexFromNumber(%u.%u) -> %u, expected %u!\n",
729 ProcNum.Group, ProcNum.Number, idxCpu, pChangeCtx->NtNumber);
730 rcNt = STATUS_INTERNAL_ERROR;
731 }
732 }
733 else
734 {
735 ProcNum.Group = 0;
736 ProcNum.Number = pChangeCtx->NtNumber;
737 }
738
739 if ( ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups)
740 && ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers))
741 {
742 if (ProcNum.Group >= g_cRtMpNtMaxGroups)
743 {
744 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range - max groups: %u!\n",
745 ProcNum.Group, ProcNum.Number, g_cRtMpNtMaxGroups);
746 rcNt = STATUS_INTERNAL_ERROR;
747 }
748
749 if (ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
750 {
751 Assert(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
752 if (g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == -1)
753 {
754 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u.%u was assigned -1 as set index!\n",
755 ProcNum.Group, ProcNum.Number);
756 rcNt = STATUS_INTERNAL_ERROR;
757 }
758
759 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
760 if (g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == NIL_RTCPUID)
761 {
762 DbgPrint("IPRT: KeProcessorAddStartNotify failure: Internal error! %u (%u.%u) translates to NIL_RTCPUID!\n",
763 pChangeCtx->NtNumber, ProcNum.Group, ProcNum.Number);
764 rcNt = STATUS_INTERNAL_ERROR;
765 }
766 }
767 else
768 {
769 DbgPrint("IPRT: KeProcessorAddStartNotify failure: max processors in group %u is %u, cannot add %u to it!\n",
770 ProcNum.Group, g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus, ProcNum.Group, ProcNum.Number);
771 rcNt = STATUS_INTERNAL_ERROR;
772 }
773 }
774 else
775 {
776 DbgPrint("IPRT: KeProcessorAddStartNotify failure: %u.%u is out of range (max %u.%u)!\n",
777 ProcNum.Group, ProcNum.Number, RT_ELEMENTS(g_aRtMpNtCpuGroups), RT_ELEMENTS(g_aRtMpNtCpuGroups[0].aidxCpuSetMembers));
778 rcNt = STATUS_INTERNAL_ERROR;
779 }
780 }
781 else
782 {
783 DbgPrint("IPRT: KeProcessorAddStartNotify failure: NtNumber=%u is outside RTCPUSET_MAX_CPUS (%u)!\n",
784 pChangeCtx->NtNumber, RTCPUSET_MAX_CPUS);
785 rcNt = STATUS_INTERNAL_ERROR;
786 }
787 if (!NT_SUCCESS(rcNt))
788 *prcOperationStatus = rcNt;
789 break;
790 }
791
792 /*
793 * Update the globals. Since we've checked out range limits and other
794 * limitations already we just AssertBreak here.
795 */
796 case KeProcessorAddCompleteNotify:
797 {
798 /*
799 * Calc the processor number and assert conditions checked in KeProcessorAddStartNotify.
800 */
801 AssertBreak(pChangeCtx->NtNumber < RTCPUSET_MAX_CPUS);
802 AssertBreak(pChangeCtx->NtNumber < g_cRtMpNtMaxCpus);
803 Assert(pChangeCtx->NtNumber == g_cRtMpNtActiveCpus); /* light assumption */
804 PROCESSOR_NUMBER ProcNum;
805 if (g_pfnrtKeGetProcessorIndexFromNumber)
806 {
807 ProcNum = pChangeCtx->ProcNumber;
808 AssertBreak(g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum) == pChangeCtx->NtNumber);
809 AssertBreak(ProcNum.Group < RT_ELEMENTS(g_aRtMpNtCpuGroups));
810 AssertBreak(ProcNum.Group < g_cRtMpNtMaxGroups);
811 }
812 else
813 {
814 ProcNum.Group = 0;
815 ProcNum.Number = pChangeCtx->NtNumber;
816 }
817 AssertBreak(ProcNum.Number < RT_ELEMENTS(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers));
818 AssertBreak(ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus);
819 AssertBreak(g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] != -1);
820 AssertBreak(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] != NIL_RTCPUID);
821
822 /*
823 * Add ourselves to the online CPU set and update the active CPU count.
824 */
825 RTCpuSetAddByIndex(&g_rtMpNtCpuSet, pChangeCtx->NtNumber);
826 ASMAtomicIncU32(&g_cRtMpNtActiveCpus);
827
828 /*
829 * Update the group info.
830 *
831 * If the index prediction failed (real hotplugging callbacks only) we
832 * have to switch it around. This is particularly annoying when we
833 * use the index as the ID.
834 */
835#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
836 RTCPUID idCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
837 RTCPUID idOld = g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber];
838 if ((idOld & ~RTMPNT_ID_F_INACTIVE) != idCpu)
839 {
840 Assert(idOld & RTMPNT_ID_F_INACTIVE);
841 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
842 g_aRtMpNtCpuGroups[rtMpCpuIdGetGroup(idOld)].aidxCpuSetMembers[rtMpCpuIdGetGroupMember(idOld)] = idxDest;
843 g_aidRtMpNtByCpuSetIdx[idxDest] = idOld;
844 }
845 g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] = idCpu;
846#else
847 Assert(g_aidRtMpNtByCpuSetIdx[pChangeCtx->NtNumber] == pChangeCtx->NtNumber);
848 int idxDest = g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
849 if ((ULONG)idxDest != pChangeCtx->NtNumber)
850 {
851 bool fFound = false;
852 uint32_t idxOldGroup = g_cRtMpNtMaxGroups;
853 while (idxOldGroup-- > 0 && !fFound)
854 {
855 uint32_t idxMember = g_aRtMpNtCpuGroups[idxOldGroup].cMaxCpus;
856 while (idxMember-- > 0)
857 if (g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] == (int)pChangeCtx->NtNumber)
858 {
859 g_aRtMpNtCpuGroups[idxOldGroup].aidxCpuSetMembers[idxMember] = idxDest;
860 fFound = true;
861 break;
862 }
863 }
864 Assert(fFound);
865 }
866#endif
867 g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] = pChangeCtx->NtNumber;
868
869 /*
870 * Do MP notification callbacks.
871 */
872 rtMpNotificationDoCallbacks(RTMPEVENT_ONLINE, pChangeCtx->NtNumber);
873 break;
874 }
875
876 case KeProcessorAddFailureNotify:
877 /* ignore */
878 break;
879
880 default:
881 AssertMsgFailed(("State=%u\n", pChangeCtx->State));
882 }
883}
884
885
886/**
887 * Wrapper around KeQueryLogicalProcessorRelationship.
888 *
889 * @returns IPRT status code.
890 * @param ppInfo Where to return the info. Pass to RTMemFree when done.
891 */
892static int rtR0NtInitQueryGroupRelations(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX **ppInfo)
893{
894 ULONG cbInfo = sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)
895 + g_cRtMpNtMaxGroups * sizeof(GROUP_RELATIONSHIP);
896 NTSTATUS rcNt;
897 do
898 {
899 SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *pInfo = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)RTMemAlloc(cbInfo);
900 if (pInfo)
901 {
902 rcNt = g_pfnrtKeQueryLogicalProcessorRelationship(NULL /*pProcNumber*/, RelationGroup, pInfo, &cbInfo);
903 if (NT_SUCCESS(rcNt))
904 {
905 *ppInfo = pInfo;
906 return VINF_SUCCESS;
907 }
908
909 RTMemFree(pInfo);
910 pInfo = NULL;
911 }
912 else
913 rcNt = STATUS_NO_MEMORY;
914 } while (rcNt == STATUS_INFO_LENGTH_MISMATCH);
915 DbgPrint("IPRT: Fatal: KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt);
916 AssertMsgFailed(("KeQueryLogicalProcessorRelationship failed: %#x\n", rcNt));
917 return RTErrConvertFromNtStatus(rcNt);
918}
919
920
921
922
923
924RTDECL(RTCPUID) RTMpCpuId(void)
925{
926 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
927
928#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
929 PROCESSOR_NUMBER ProcNum;
930 ProcNum.Group = 0;
931 if (g_pfnrtKeGetCurrentProcessorNumberEx)
932 {
933 ProcNum.Number = 0;
934 g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
935 }
936 else
937 ProcNum.Number = KeGetCurrentProcessorNumber(); /* Number is 8-bit, so we're not subject to BYTE -> WORD upgrade in WDK. */
938 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
939
940#else
941
942 if (g_pfnrtKeGetCurrentProcessorNumberEx)
943 {
944 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
945 Assert(idxCpu < RTCPUSET_MAX_CPUS);
946 return idxCpu;
947 }
948
949 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
950#endif
951}
952
953
954RTDECL(int) RTMpCurSetIndex(void)
955{
956#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
957 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
958
959 if (g_pfnrtKeGetCurrentProcessorNumberEx)
960 {
961 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(NULL);
962 Assert(idxCpu < RTCPUSET_MAX_CPUS);
963 return idxCpu;
964 }
965 return (uint8_t)KeGetCurrentProcessorNumber(); /* PCR->Number was changed from BYTE to WORD in the WDK, thus the cast. */
966#else
967 return (int)RTMpCpuId();
968#endif
969}
970
971
972RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
973{
974#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
975 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
976
977 PROCESSOR_NUMBER ProcNum = { 0 , 0, 0 };
978 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetCurrentProcessorNumberEx(&ProcNum);
979 Assert(idxCpu < RTCPUSET_MAX_CPUS);
980 *pidCpu = RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
981 return idxCpu;
982#else
983 return *pidCpu = RTMpCpuId();
984#endif
985}
986
987
988RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
989{
990#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
991 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
992
993 if (idCpu != NIL_RTCPUID)
994 {
995 if (g_pfnrtKeGetProcessorIndexFromNumber)
996 {
997 PROCESSOR_NUMBER ProcNum;
998 ProcNum.Group = rtMpCpuIdGetGroup(idCpu);
999 ProcNum.Number = rtMpCpuIdGetGroupMember(idCpu);
1000 ProcNum.Reserved = 0;
1001 KEPROCESSORINDEX idxCpu = g_pfnrtKeGetProcessorIndexFromNumber(&ProcNum);
1002 if (idxCpu != INVALID_PROCESSOR_INDEX)
1003 {
1004 Assert(idxCpu < g_cRtMpNtMaxCpus);
1005 Assert((ULONG)g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number] == idxCpu);
1006 return idxCpu;
1007 }
1008
1009 /* Since NT assigned indexes as the CPUs come online, we cannot produce an ID <-> index
1010 mapping for not-yet-onlined CPUS that is consistent. We just have to do our best... */
1011 if ( ProcNum.Group < g_cRtMpNtMaxGroups
1012 && ProcNum.Number < g_aRtMpNtCpuGroups[ProcNum.Group].cMaxCpus)
1013 return g_aRtMpNtCpuGroups[ProcNum.Group].aidxCpuSetMembers[ProcNum.Number];
1014 }
1015 else if (rtMpCpuIdGetGroup(idCpu) == 0)
1016 return rtMpCpuIdGetGroupMember(idCpu);
1017 }
1018 return -1;
1019#else
1020 /* 1:1 mapping, just do range checks. */
1021 return idCpu < RTCPUSET_MAX_CPUS ? (int)idCpu : -1;
1022#endif
1023}
1024
1025
1026RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
1027{
1028#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1029 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1030
1031 if ((unsigned)iCpu < g_cRtMpNtMaxCpus)
1032 {
1033 if (g_pfnrtKeGetProcessorIndexFromNumber)
1034 {
1035 PROCESSOR_NUMBER ProcNum = { 0, 0, 0 };
1036 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(iCpu, &ProcNum);
1037 if (NT_SUCCESS(rcNt))
1038 {
1039 Assert(ProcNum.Group <= g_cRtMpNtMaxGroups);
1040 Assert( (g_aidRtMpNtByCpuSetIdx[iCpu] & ~RTMPNT_ID_F_INACTIVE)
1041 == RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number));
1042 return RTMPCPUID_FROM_GROUP_AND_NUMBER(ProcNum.Group, ProcNum.Number);
1043 }
1044 }
1045 return g_aidRtMpNtByCpuSetIdx[iCpu];
1046 }
1047 return NIL_RTCPUID;
1048#else
1049 /* 1:1 mapping, just do range checks. */
1050 return (unsigned)iCpu < RTCPUSET_MAX_CPUS ? iCpu : NIL_RTCPUID;
1051#endif
1052}
1053
1054
1055RTDECL(int) RTMpSetIndexFromCpuGroupMember(uint32_t idxGroup, uint32_t idxMember)
1056{
1057 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1058
1059 if (idxGroup < g_cRtMpNtMaxGroups)
1060 if (idxMember < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus)
1061 return g_aRtMpNtCpuGroups[idxGroup].aidxCpuSetMembers[idxMember];
1062 return -1;
1063}
1064
1065
1066RTDECL(uint32_t) RTMpGetCpuGroupCounts(uint32_t idxGroup, uint32_t *pcActive)
1067{
1068 if (idxGroup < g_cRtMpNtMaxGroups)
1069 {
1070 if (pcActive)
1071 *pcActive = g_aRtMpNtCpuGroups[idxGroup].cActiveCpus;
1072 return g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1073 }
1074 if (pcActive)
1075 *pcActive = 0;
1076 return 0;
1077}
1078
1079
1080RTDECL(uint32_t) RTMpGetMaxCpuGroupCount(void)
1081{
1082 return g_cRtMpNtMaxGroups;
1083}
1084
1085
1086RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
1087{
1088 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1089
1090#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1091 return RTMPCPUID_FROM_GROUP_AND_NUMBER(g_cRtMpNtMaxGroups - 1, g_aRtMpNtCpuGroups[g_cRtMpNtMaxGroups - 1].cMaxCpus - 1);
1092#else
1093 /* According to MSDN the processor indexes goes from 0 to the maximum
1094 number of CPUs in the system. We've check this in initterm-r0drv-nt.cpp. */
1095 return g_cRtMpNtMaxCpus - 1;
1096#endif
1097}
1098
1099
1100RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
1101{
1102 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1103 return RTCpuSetIsMember(&g_rtMpNtCpuSet, idCpu);
1104}
1105
1106
1107RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
1108{
1109 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1110
1111#ifdef IPRT_WITH_RTCPUID_AS_GROUP_AND_NUMBER
1112 if (idCpu != NIL_RTCPUID)
1113 {
1114 unsigned idxGroup = rtMpCpuIdGetGroup(idCpu);
1115 if (idxGroup < g_cRtMpNtMaxGroups)
1116 return rtMpCpuIdGetGroupMember(idCpu) < g_aRtMpNtCpuGroups[idxGroup].cMaxCpus;
1117 }
1118 return false;
1119
1120#else
1121 /* A possible CPU ID is one with a value lower than g_cRtMpNtMaxCpus (see
1122 comment in RTMpGetMaxCpuId). */
1123 return idCpu < g_cRtMpNtMaxCpus;
1124#endif
1125}
1126
1127
1128
1129RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
1130{
1131 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1132
1133 /* The set of possible CPU IDs(/indexes) are from 0 up to
1134 g_cRtMpNtMaxCpus (see comment in RTMpGetMaxCpuId). */
1135 RTCpuSetEmpty(pSet);
1136 int idxCpu = g_cRtMpNtMaxCpus;
1137 while (idxCpu-- > 0)
1138 RTCpuSetAddByIndex(pSet, idxCpu);
1139 return pSet;
1140}
1141
1142
1143RTDECL(RTCPUID) RTMpGetCount(void)
1144{
1145 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1146 return g_cRtMpNtMaxCpus;
1147}
1148
1149
1150RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
1151{
1152 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1153
1154 *pSet = g_rtMpNtCpuSet;
1155 return pSet;
1156}
1157
1158
1159RTDECL(RTCPUID) RTMpGetOnlineCount(void)
1160{
1161 RTCPUSET Set;
1162 RTMpGetOnlineSet(&Set);
1163 return RTCpuSetCount(&Set);
1164}
1165
1166
1167RTDECL(RTCPUID) RTMpGetOnlineCoreCount(void)
1168{
1169 /** @todo fix me */
1170 return RTMpGetOnlineCount();
1171}
1172
1173
1174
1175#if 0
1176/* Experiment with checking the undocumented KPRCB structure
1177 * 'dt nt!_kprcb 0xaddress' shows the layout
1178 */
1179typedef struct
1180{
1181 LIST_ENTRY DpcListHead;
1182 ULONG_PTR DpcLock;
1183 volatile ULONG DpcQueueDepth;
1184 ULONG DpcQueueCount;
1185} KDPC_DATA, *PKDPC_DATA;
1186
1187RTDECL(bool) RTMpIsCpuWorkPending(void)
1188{
1189 uint8_t *pkprcb;
1190 PKDPC_DATA pDpcData;
1191
1192 _asm {
1193 mov eax, fs:0x20
1194 mov pkprcb, eax
1195 }
1196 pDpcData = (PKDPC_DATA)(pkprcb + 0x19e0);
1197 if (pDpcData->DpcQueueDepth)
1198 return true;
1199
1200 pDpcData++;
1201 if (pDpcData->DpcQueueDepth)
1202 return true;
1203 return false;
1204}
1205#else
1206RTDECL(bool) RTMpIsCpuWorkPending(void)
1207{
1208 /** @todo not implemented */
1209 return false;
1210}
1211#endif
1212
1213
1214/**
1215 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1216 * the RTMpOnAll case.
1217 *
1218 * @param uUserCtx The user context argument (PRTMPARGS).
1219 */
1220static ULONG_PTR rtmpNtOnAllBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1221{
1222 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1223 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1224 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1225 return 0;
1226}
1227
1228
1229/**
1230 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1231 * the RTMpOnOthers case.
1232 *
1233 * @param uUserCtx The user context argument (PRTMPARGS).
1234 */
1235static ULONG_PTR rtmpNtOnOthersBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1236{
1237 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1238 RTCPUID idCpu = RTMpCpuId();
1239 if (pArgs->idCpu != idCpu)
1240 {
1241 /*ASMAtomicIncU32(&pArgs->cHits); - not needed */
1242 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1243 }
1244 return 0;
1245}
1246
1247
1248/**
1249 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1250 * the RTMpOnPair case.
1251 *
1252 * @param uUserCtx The user context argument (PRTMPARGS).
1253 */
1254static ULONG_PTR rtmpNtOnPairBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1255{
1256 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1257 RTCPUID idCpu = RTMpCpuId();
1258 if ( pArgs->idCpu == idCpu
1259 || pArgs->idCpu2 == idCpu)
1260 {
1261 ASMAtomicIncU32(&pArgs->cHits);
1262 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1263 }
1264 return 0;
1265}
1266
1267
1268/**
1269 * Wrapper between the native KIPI_BROADCAST_WORKER and IPRT's PFNRTMPWORKER for
1270 * the RTMpOnSpecific case.
1271 *
1272 * @param uUserCtx The user context argument (PRTMPARGS).
1273 */
1274static ULONG_PTR rtmpNtOnSpecificBroadcastIpiWrapper(ULONG_PTR uUserCtx)
1275{
1276 PRTMPARGS pArgs = (PRTMPARGS)uUserCtx;
1277 RTCPUID idCpu = RTMpCpuId();
1278 if (pArgs->idCpu == idCpu)
1279 {
1280 ASMAtomicIncU32(&pArgs->cHits);
1281 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
1282 }
1283 return 0;
1284}
1285
1286
1287/**
1288 * Internal worker for the RTMpOn* APIs using KeIpiGenericCall.
1289 *
1290 * @returns VINF_SUCCESS.
1291 * @param pfnWorker The callback.
1292 * @param pvUser1 User argument 1.
1293 * @param pvUser2 User argument 2.
1294 * @param pfnNativeWrapper The wrapper between the NT and IPRT callbacks.
1295 * @param idCpu First CPU to match, ultimately specific to the
1296 * pfnNativeWrapper used.
1297 * @param idCpu2 Second CPU to match, ultimately specific to the
1298 * pfnNativeWrapper used.
1299 * @param pcHits Where to return the number of this. Optional.
1300 */
1301static int rtMpCallUsingBroadcastIpi(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1302 PKIPI_BROADCAST_WORKER pfnNativeWrapper, RTCPUID idCpu, RTCPUID idCpu2,
1303 uint32_t *pcHits)
1304{
1305 RTMPARGS Args;
1306 Args.pfnWorker = pfnWorker;
1307 Args.pvUser1 = pvUser1;
1308 Args.pvUser2 = pvUser2;
1309 Args.idCpu = idCpu;
1310 Args.idCpu2 = idCpu2;
1311 Args.cRefs = 0;
1312 Args.cHits = 0;
1313
1314 AssertPtr(g_pfnrtKeIpiGenericCall);
1315 g_pfnrtKeIpiGenericCall(pfnNativeWrapper, (uintptr_t)&Args);
1316 if (pcHits)
1317 *pcHits = Args.cHits;
1318 return VINF_SUCCESS;
1319}
1320
1321
1322/**
1323 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1324 *
1325 * @param Dpc DPC object
1326 * @param DeferredContext Context argument specified by KeInitializeDpc
1327 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1328 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1329 */
1330static VOID rtmpNtDPCWrapper(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1331{
1332 PRTMPARGS pArgs = (PRTMPARGS)DeferredContext;
1333 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1334
1335 ASMAtomicIncU32(&pArgs->cHits);
1336 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
1337
1338 /* Dereference the argument structure. */
1339 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1340 Assert(cRefs >= 0);
1341 if (cRefs == 0)
1342 RTMemFree(pArgs);
1343}
1344
1345
1346/**
1347 * Wrapper around KeSetTargetProcessorDpcEx / KeSetTargetProcessorDpc.
1348 *
1349 * This is shared with the timer code.
1350 *
1351 * @returns IPRT status code (errors are asserted).
1352 * @param pDpc The DPC.
1353 * @param idCpu The ID of the new target CPU.
1354 */
1355DECLHIDDEN(int) rtMpNtSetTargetProcessorDpc(KDPC *pDpc, RTCPUID idCpu)
1356{
1357 if (g_pfnrtKeSetTargetProcessorDpcEx)
1358 {
1359 /* Convert to stupid process number (bet KeSetTargetProcessorDpcEx does
1360 the reverse conversion internally). */
1361 PROCESSOR_NUMBER ProcNum;
1362 NTSTATUS rcNt = g_pfnrtKeGetProcessorNumberFromIndex(RTMpCpuIdToSetIndex(idCpu), &ProcNum);
1363 AssertLogRelMsgReturn(NT_SUCCESS(rcNt), ("KeGetProcessorNumberFromIndex(%u) -> %#x\n", idCpu, rcNt),
1364 RTErrConvertFromNtStatus(rcNt));
1365
1366 rcNt = g_pfnrtKeSetTargetProcessorDpcEx(pDpc, &ProcNum);
1367 AssertLogRelMsgReturn(NT_SUCCESS(rcNt),
1368 ("KeSetTargetProcessorDpcEx(,%u(%u/%u)) -> %#x\n", idCpu, ProcNum.Group, ProcNum.Number, rcNt),
1369 RTErrConvertFromNtStatus(rcNt));
1370 }
1371 else if (g_pfnrtKeSetTargetProcessorDpc)
1372 g_pfnrtKeSetTargetProcessorDpc(pDpc, RTMpCpuIdToSetIndex(idCpu));
1373 else
1374 return VERR_NOT_SUPPORTED;
1375 return VINF_SUCCESS;
1376}
1377
1378
1379/**
1380 * Internal worker for the RTMpOn* APIs.
1381 *
1382 * @returns IPRT status code.
1383 * @param pfnWorker The callback.
1384 * @param pvUser1 User argument 1.
1385 * @param pvUser2 User argument 2.
1386 * @param enmCpuid What to do / is idCpu valid.
1387 * @param idCpu Used if enmCpuid is RT_NT_CPUID_SPECIFIC or
1388 * RT_NT_CPUID_PAIR, otherwise ignored.
1389 * @param idCpu2 Used if enmCpuid is RT_NT_CPUID_PAIR, otherwise ignored.
1390 * @param pcHits Where to return the number of this. Optional.
1391 */
1392static int rtMpCallUsingDpcs(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2,
1393 RT_NT_CPUID enmCpuid, RTCPUID idCpu, RTCPUID idCpu2, uint32_t *pcHits)
1394{
1395#if 0
1396 /* KeFlushQueuedDpcs must be run at IRQL PASSIVE_LEVEL according to MSDN, but the
1397 * driver verifier doesn't complain...
1398 */
1399 AssertMsg(KeGetCurrentIrql() == PASSIVE_LEVEL, ("%d != %d (PASSIVE_LEVEL)\n", KeGetCurrentIrql(), PASSIVE_LEVEL));
1400#endif
1401 /* KeFlushQueuedDpcs is not present in Windows 2000; import it dynamically so we can just fail this call. */
1402 if (!g_pfnrtNtKeFlushQueuedDpcs)
1403 return VERR_NOT_SUPPORTED;
1404
1405 /*
1406 * Make a copy of the active CPU set and figure out how many KDPCs we really need.
1407 * We must not try setup DPCs for CPUs which aren't there, because that may fail.
1408 */
1409 RTCPUSET OnlineSet = g_rtMpNtCpuSet;
1410 uint32_t cDpcsNeeded;
1411 switch (enmCpuid)
1412 {
1413 case RT_NT_CPUID_SPECIFIC:
1414 cDpcsNeeded = 1;
1415 break;
1416 case RT_NT_CPUID_PAIR:
1417 cDpcsNeeded = 2;
1418 break;
1419 default:
1420 do
1421 {
1422 cDpcsNeeded = g_cRtMpNtActiveCpus;
1423 OnlineSet = g_rtMpNtCpuSet;
1424 } while (cDpcsNeeded != g_cRtMpNtActiveCpus);
1425 break;
1426 }
1427
1428 /*
1429 * Allocate an RTMPARGS structure followed by cDpcsNeeded KDPCs
1430 * and initialize them.
1431 */
1432 PRTMPARGS pArgs = (PRTMPARGS)RTMemAllocZ(sizeof(RTMPARGS) + cDpcsNeeded * sizeof(KDPC));
1433 if (!pArgs)
1434 return VERR_NO_MEMORY;
1435
1436 pArgs->pfnWorker = pfnWorker;
1437 pArgs->pvUser1 = pvUser1;
1438 pArgs->pvUser2 = pvUser2;
1439 pArgs->idCpu = NIL_RTCPUID;
1440 pArgs->idCpu2 = NIL_RTCPUID;
1441 pArgs->cHits = 0;
1442 pArgs->cRefs = 1;
1443
1444 int rc;
1445 KDPC *paExecCpuDpcs = (KDPC *)(pArgs + 1);
1446 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1447 {
1448 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1449 if (g_pfnrtKeSetImportanceDpc)
1450 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1451 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1452 pArgs->idCpu = idCpu;
1453 }
1454 else if (enmCpuid == RT_NT_CPUID_PAIR)
1455 {
1456 KeInitializeDpc(&paExecCpuDpcs[0], rtmpNtDPCWrapper, pArgs);
1457 if (g_pfnrtKeSetImportanceDpc)
1458 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[0], HighImportance);
1459 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[0], idCpu);
1460 pArgs->idCpu = idCpu;
1461
1462 KeInitializeDpc(&paExecCpuDpcs[1], rtmpNtDPCWrapper, pArgs);
1463 if (g_pfnrtKeSetImportanceDpc)
1464 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[1], HighImportance);
1465 if (RT_SUCCESS(rc))
1466 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[1], (int)idCpu2);
1467 pArgs->idCpu2 = idCpu2;
1468 }
1469 else
1470 {
1471 rc = VINF_SUCCESS;
1472 for (uint32_t i = 0; i < cDpcsNeeded && RT_SUCCESS(rc); i++)
1473 if (RTCpuSetIsMemberByIndex(&OnlineSet, i))
1474 {
1475 KeInitializeDpc(&paExecCpuDpcs[i], rtmpNtDPCWrapper, pArgs);
1476 if (g_pfnrtKeSetImportanceDpc)
1477 g_pfnrtKeSetImportanceDpc(&paExecCpuDpcs[i], HighImportance);
1478 rc = rtMpNtSetTargetProcessorDpc(&paExecCpuDpcs[i], RTMpCpuIdFromSetIndex(i));
1479 }
1480 }
1481 if (RT_FAILURE(rc))
1482 {
1483 RTMemFree(pArgs);
1484 return rc;
1485 }
1486
1487 /*
1488 * Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1489 * KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL.
1490 */
1491 KIRQL oldIrql;
1492 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1493
1494 /*
1495 * We cannot do other than assume a 1:1 relationship between the
1496 * affinity mask and the process despite the warnings in the docs.
1497 * If someone knows a better way to get this done, please let bird know.
1498 */
1499 ASMCompilerBarrier(); /* paranoia */
1500 if (enmCpuid == RT_NT_CPUID_SPECIFIC)
1501 {
1502 ASMAtomicIncS32(&pArgs->cRefs);
1503 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1504 Assert(fRc); NOREF(fRc);
1505 }
1506 else if (enmCpuid == RT_NT_CPUID_PAIR)
1507 {
1508 ASMAtomicIncS32(&pArgs->cRefs);
1509 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[0], 0, 0);
1510 Assert(fRc); NOREF(fRc);
1511
1512 ASMAtomicIncS32(&pArgs->cRefs);
1513 fRc = KeInsertQueueDpc(&paExecCpuDpcs[1], 0, 0);
1514 Assert(fRc); NOREF(fRc);
1515 }
1516 else
1517 {
1518 uint32_t iSelf = RTMpCurSetIndex();
1519 for (uint32_t i = 0; i < cDpcsNeeded; i++)
1520 {
1521 if ( (i != iSelf)
1522 && RTCpuSetIsMemberByIndex(&OnlineSet, i))
1523 {
1524 ASMAtomicIncS32(&pArgs->cRefs);
1525 BOOLEAN fRc = KeInsertQueueDpc(&paExecCpuDpcs[i], 0, 0);
1526 Assert(fRc); NOREF(fRc);
1527 }
1528 }
1529 if (enmCpuid != RT_NT_CPUID_OTHERS)
1530 pfnWorker(iSelf, pvUser1, pvUser2);
1531 }
1532
1533 KeLowerIrql(oldIrql);
1534
1535 /*
1536 * Flush all DPCs and wait for completion. (can take long!)
1537 */
1538 /** @todo Consider changing this to an active wait using some atomic inc/dec
1539 * stuff (and check for the current cpu above in the specific case). */
1540 /** @todo Seems KeFlushQueuedDpcs doesn't wait for the DPCs to be completely
1541 * executed. Seen pArgs being freed while some CPU was using it before
1542 * cRefs was added. */
1543 if (g_pfnrtNtKeFlushQueuedDpcs)
1544 g_pfnrtNtKeFlushQueuedDpcs();
1545
1546 if (pcHits)
1547 *pcHits = pArgs->cHits;
1548
1549 /* Dereference the argument structure. */
1550 int32_t cRefs = ASMAtomicDecS32(&pArgs->cRefs);
1551 Assert(cRefs >= 0);
1552 if (cRefs == 0)
1553 RTMemFree(pArgs);
1554
1555 return VINF_SUCCESS;
1556}
1557
1558
1559RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1560{
1561 if (g_pfnrtKeIpiGenericCall)
1562 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnAllBroadcastIpiWrapper,
1563 NIL_RTCPUID, NIL_RTCPUID, NULL);
1564 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_ALL, NIL_RTCPUID, NIL_RTCPUID, NULL);
1565}
1566
1567
1568RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1569{
1570 if (g_pfnrtKeIpiGenericCall)
1571 return rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnOthersBroadcastIpiWrapper,
1572 NIL_RTCPUID, NIL_RTCPUID, NULL);
1573 return rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_OTHERS, NIL_RTCPUID, NIL_RTCPUID, NULL);
1574}
1575
1576
1577RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1578{
1579 int rc;
1580 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
1581 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
1582 if ((fFlags & RTMPON_F_CONCURRENT_EXEC) && !g_pfnrtKeIpiGenericCall)
1583 return VERR_NOT_SUPPORTED;
1584
1585 /*
1586 * Check that both CPUs are online before doing the broadcast call.
1587 */
1588 if ( RTMpIsCpuOnline(idCpu1)
1589 && RTMpIsCpuOnline(idCpu2))
1590 {
1591 /*
1592 * The broadcast IPI isn't quite as bad as it could have been, because
1593 * it looks like windows doesn't synchronize CPUs on the way out, they
1594 * seems to get back to normal work while the pair is still busy.
1595 */
1596 uint32_t cHits = 0;
1597 if (g_pfnrtKeIpiGenericCall)
1598 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnPairBroadcastIpiWrapper, idCpu1, idCpu2, &cHits);
1599 else
1600 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_PAIR, idCpu1, idCpu2, &cHits);
1601 if (RT_SUCCESS(rc))
1602 {
1603 Assert(cHits <= 2);
1604 if (cHits == 2)
1605 rc = VINF_SUCCESS;
1606 else if (cHits == 1)
1607 rc = VERR_NOT_ALL_CPUS_SHOWED;
1608 else if (cHits == 0)
1609 rc = VERR_CPU_OFFLINE;
1610 else
1611 rc = VERR_CPU_IPE_1;
1612 }
1613 }
1614 /*
1615 * A CPU must be present to be considered just offline.
1616 */
1617 else if ( RTMpIsCpuPresent(idCpu1)
1618 && RTMpIsCpuPresent(idCpu2))
1619 rc = VERR_CPU_OFFLINE;
1620 else
1621 rc = VERR_CPU_NOT_FOUND;
1622 return rc;
1623}
1624
1625
1626RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
1627{
1628 return g_pfnrtKeIpiGenericCall != NULL;
1629}
1630
1631
1632/**
1633 * Releases a reference to a RTMPNTONSPECIFICARGS heap allocation, freeing it
1634 * when the last reference is released.
1635 */
1636DECLINLINE(void) rtMpNtOnSpecificRelease(PRTMPNTONSPECIFICARGS pArgs)
1637{
1638 uint32_t cRefs = ASMAtomicDecU32(&pArgs->cRefs);
1639 AssertMsg(cRefs <= 1, ("cRefs=%#x\n", cRefs));
1640 if (cRefs == 0)
1641 RTMemFree(pArgs);
1642}
1643
1644
1645/**
1646 * Wrapper between the native nt per-cpu callbacks and PFNRTWORKER
1647 *
1648 * @param Dpc DPC object
1649 * @param DeferredContext Context argument specified by KeInitializeDpc
1650 * @param SystemArgument1 Argument specified by KeInsertQueueDpc
1651 * @param SystemArgument2 Argument specified by KeInsertQueueDpc
1652 */
1653static VOID rtMpNtOnSpecificDpcWrapper(IN PKDPC Dpc, IN PVOID DeferredContext,
1654 IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1655{
1656 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)DeferredContext;
1657 RT_NOREF3(Dpc, SystemArgument1, SystemArgument2);
1658
1659 ASMAtomicWriteBool(&pArgs->fExecuting, true);
1660
1661 pArgs->CallbackArgs.pfnWorker(RTMpCpuId(), pArgs->CallbackArgs.pvUser1, pArgs->CallbackArgs.pvUser2);
1662
1663 ASMAtomicWriteBool(&pArgs->fDone, true);
1664 KeSetEvent(&pArgs->DoneEvt, 1 /*PriorityIncrement*/, FALSE /*Wait*/);
1665
1666 rtMpNtOnSpecificRelease(pArgs);
1667}
1668
1669
1670RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
1671{
1672 /*
1673 * Don't try mess with an offline CPU.
1674 */
1675 if (!RTMpIsCpuOnline(idCpu))
1676 return !RTMpIsCpuPossible(idCpu)
1677 ? VERR_CPU_NOT_FOUND
1678 : VERR_CPU_OFFLINE;
1679
1680 /*
1681 * Use the broadcast IPI routine if there are no more than two CPUs online,
1682 * or if the current IRQL is unsuitable for KeWaitForSingleObject.
1683 */
1684 int rc;
1685 uint32_t cHits = 0;
1686 if ( g_pfnrtKeIpiGenericCall
1687 && ( RTMpGetOnlineCount() <= 2
1688 || KeGetCurrentIrql() > APC_LEVEL)
1689 )
1690 {
1691 rc = rtMpCallUsingBroadcastIpi(pfnWorker, pvUser1, pvUser2, rtmpNtOnSpecificBroadcastIpiWrapper,
1692 idCpu, NIL_RTCPUID, &cHits);
1693 if (RT_SUCCESS(rc))
1694 {
1695 if (cHits == 1)
1696 return VINF_SUCCESS;
1697 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1698 }
1699 return rc;
1700 }
1701
1702#if 0
1703 rc = rtMpCallUsingDpcs(pfnWorker, pvUser1, pvUser2, RT_NT_CPUID_SPECIFIC, idCpu, NIL_RTCPUID, &cHits);
1704 if (RT_SUCCESS(rc))
1705 {
1706 if (cHits == 1)
1707 return VINF_SUCCESS;
1708 rc = cHits == 0 ? VERR_CPU_OFFLINE : VERR_CPU_IPE_1;
1709 }
1710 return rc;
1711
1712#else
1713 /*
1714 * Initialize the argument package and the objects within it.
1715 * The package is referenced counted to avoid unnecessary spinning to
1716 * synchronize cleanup and prevent stack corruption.
1717 */
1718 PRTMPNTONSPECIFICARGS pArgs = (PRTMPNTONSPECIFICARGS)RTMemAllocZ(sizeof(*pArgs));
1719 if (!pArgs)
1720 return VERR_NO_MEMORY;
1721 pArgs->cRefs = 2;
1722 pArgs->fExecuting = false;
1723 pArgs->fDone = false;
1724 pArgs->CallbackArgs.pfnWorker = pfnWorker;
1725 pArgs->CallbackArgs.pvUser1 = pvUser1;
1726 pArgs->CallbackArgs.pvUser2 = pvUser2;
1727 pArgs->CallbackArgs.idCpu = idCpu;
1728 pArgs->CallbackArgs.cHits = 0;
1729 pArgs->CallbackArgs.cRefs = 2;
1730 KeInitializeEvent(&pArgs->DoneEvt, SynchronizationEvent, FALSE /* not signalled */);
1731 KeInitializeDpc(&pArgs->Dpc, rtMpNtOnSpecificDpcWrapper, pArgs);
1732 if (g_pfnrtKeSetImportanceDpc)
1733 g_pfnrtKeSetImportanceDpc(&pArgs->Dpc, HighImportance);
1734 rc = rtMpNtSetTargetProcessorDpc(&pArgs->Dpc, idCpu);
1735 if (RT_FAILURE(rc))
1736 {
1737 RTMemFree(pArgs);
1738 return rc;
1739 }
1740
1741 /*
1742 * Disable preemption while we check the current processor and inserts the DPC.
1743 */
1744 KIRQL bOldIrql;
1745 KeRaiseIrql(DISPATCH_LEVEL, &bOldIrql);
1746 ASMCompilerBarrier(); /* paranoia */
1747
1748 if (RTMpCpuId() == idCpu)
1749 {
1750 /* Just execute the callback on the current CPU. */
1751 pfnWorker(idCpu, pvUser1, pvUser2);
1752 KeLowerIrql(bOldIrql);
1753
1754 RTMemFree(pArgs);
1755 return VINF_SUCCESS;
1756 }
1757
1758 /* Different CPU, so queue it if the CPU is still online. */
1759 if (RTMpIsCpuOnline(idCpu))
1760 {
1761 BOOLEAN fRc = KeInsertQueueDpc(&pArgs->Dpc, 0, 0);
1762 Assert(fRc); NOREF(fRc);
1763 KeLowerIrql(bOldIrql);
1764
1765 uint64_t const nsRealWaitTS = RTTimeNanoTS();
1766
1767 /*
1768 * Wait actively for a while in case the CPU/thread responds quickly.
1769 */
1770 uint32_t cLoopsLeft = 0x20000;
1771 while (cLoopsLeft-- > 0)
1772 {
1773 if (pArgs->fDone)
1774 {
1775 rtMpNtOnSpecificRelease(pArgs);
1776 return VINF_SUCCESS;
1777 }
1778 ASMNopPause();
1779 }
1780
1781 /*
1782 * It didn't respond, so wait on the event object, poking the CPU if it's slow.
1783 */
1784 LARGE_INTEGER Timeout;
1785 Timeout.QuadPart = -10000; /* 1ms */
1786 NTSTATUS rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1787 if (rcNt == STATUS_SUCCESS)
1788 {
1789 rtMpNtOnSpecificRelease(pArgs);
1790 return VINF_SUCCESS;
1791 }
1792
1793 /* If it hasn't respondend yet, maybe poke it and wait some more. */
1794 if (rcNt == STATUS_TIMEOUT)
1795 {
1796 if ( !pArgs->fExecuting
1797 && ( g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiW7Plus
1798 || g_pfnrtMpPokeCpuWorker == rtMpPokeCpuUsingHalRequestIpiPreW7))
1799 RTMpPokeCpu(idCpu);
1800
1801 Timeout.QuadPart = -1280000; /* 128ms */
1802 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1803 if (rcNt == STATUS_SUCCESS)
1804 {
1805 rtMpNtOnSpecificRelease(pArgs);
1806 return VINF_SUCCESS;
1807 }
1808 }
1809
1810 /*
1811 * Something weird is happening, try bail out.
1812 */
1813 if (KeRemoveQueueDpc(&pArgs->Dpc))
1814 {
1815 RTMemFree(pArgs); /* DPC was still queued, so we can return without further ado. */
1816 LogRel(("RTMpOnSpecific(%#x): Not processed after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1817 }
1818 else
1819 {
1820 /* DPC is running, wait a good while for it to complete. */
1821 LogRel(("RTMpOnSpecific(%#x): Still running after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1822
1823 Timeout.QuadPart = -30*1000*1000*10; /* 30 seconds */
1824 rcNt = KeWaitForSingleObject(&pArgs->DoneEvt, Executive, KernelMode, FALSE /* Alertable */, &Timeout);
1825 if (rcNt != STATUS_SUCCESS)
1826 LogRel(("RTMpOnSpecific(%#x): Giving up on running worker after %llu ns: rcNt=%#x\n", idCpu, RTTimeNanoTS() - nsRealWaitTS, rcNt));
1827 }
1828 rc = RTErrConvertFromNtStatus(rcNt);
1829 }
1830 else
1831 {
1832 /* CPU is offline.*/
1833 KeLowerIrql(bOldIrql);
1834 rc = !RTMpIsCpuPossible(idCpu) ? VERR_CPU_NOT_FOUND : VERR_CPU_OFFLINE;
1835 }
1836
1837 rtMpNtOnSpecificRelease(pArgs);
1838 return rc;
1839#endif
1840}
1841
1842
1843
1844
1845static VOID rtMpNtPokeCpuDummy(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
1846{
1847 NOREF(Dpc);
1848 NOREF(DeferredContext);
1849 NOREF(SystemArgument1);
1850 NOREF(SystemArgument2);
1851}
1852
1853
1854/** Callback used by rtMpPokeCpuUsingBroadcastIpi. */
1855static ULONG_PTR rtMpIpiGenericCall(ULONG_PTR Argument)
1856{
1857 NOREF(Argument);
1858 return 0;
1859}
1860
1861
1862/**
1863 * RTMpPokeCpu worker that uses broadcast IPIs for doing the work.
1864 *
1865 * @returns VINF_SUCCESS
1866 * @param idCpu The CPU identifier.
1867 */
1868int rtMpPokeCpuUsingBroadcastIpi(RTCPUID idCpu)
1869{
1870 NOREF(idCpu);
1871 g_pfnrtKeIpiGenericCall(rtMpIpiGenericCall, 0);
1872 return VINF_SUCCESS;
1873}
1874
1875
1876/**
1877 * RTMpPokeCpu worker that uses the Windows 7 and later version of
1878 * HalRequestIpip to get the job done.
1879 *
1880 * @returns VINF_SUCCESS
1881 * @param idCpu The CPU identifier.
1882 */
1883int rtMpPokeCpuUsingHalRequestIpiW7Plus(RTCPUID idCpu)
1884{
1885 /* idCpu is an HAL processor index, so we can use it directly. */
1886 PKAFFINITY_EX pTarget = (PKAFFINITY_EX)alloca(g_cbRtMpNtKaffinityEx);
1887 pTarget->Size = g_cRtMpNtKaffinityExEntries; /* (just in case KeInitializeAffinityEx starts using it) */
1888 g_pfnrtKeInitializeAffinityEx(pTarget);
1889 g_pfnrtKeAddProcessorAffinityEx(pTarget, idCpu);
1890
1891 g_pfnrtHalRequestIpiW7Plus(0, pTarget);
1892 return VINF_SUCCESS;
1893}
1894
1895
1896/**
1897 * RTMpPokeCpu worker that uses the Vista and earlier version of HalRequestIpip
1898 * to get the job done.
1899 *
1900 * @returns VINF_SUCCESS
1901 * @param idCpu The CPU identifier.
1902 */
1903int rtMpPokeCpuUsingHalRequestIpiPreW7(RTCPUID idCpu)
1904{
1905 __debugbreak(); /** @todo this code needs testing!! */
1906 KAFFINITY Target = 1;
1907 Target <<= idCpu;
1908 g_pfnrtHalRequestIpiPreW7(Target);
1909 return VINF_SUCCESS;
1910}
1911
1912
1913int rtMpPokeCpuUsingFailureNotSupported(RTCPUID idCpu)
1914{
1915 NOREF(idCpu);
1916 return VERR_NOT_SUPPORTED;
1917}
1918
1919
1920int rtMpPokeCpuUsingDpc(RTCPUID idCpu)
1921{
1922 Assert(g_cRtMpNtMaxCpus > 0 && g_cRtMpNtMaxGroups > 0); /* init order */
1923
1924 /*
1925 * APC fallback.
1926 */
1927 static KDPC s_aPokeDpcs[RTCPUSET_MAX_CPUS] = {0};
1928 static bool s_fPokeDPCsInitialized = false;
1929
1930 if (!s_fPokeDPCsInitialized)
1931 {
1932 for (unsigned i = 0; i < g_cRtMpNtMaxCpus; i++)
1933 {
1934 KeInitializeDpc(&s_aPokeDpcs[i], rtMpNtPokeCpuDummy, NULL);
1935 if (g_pfnrtKeSetImportanceDpc)
1936 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[i], HighImportance);
1937 int rc = rtMpNtSetTargetProcessorDpc(&s_aPokeDpcs[i], idCpu);
1938 if (RT_FAILURE(rc))
1939 return rc;
1940 }
1941
1942 s_fPokeDPCsInitialized = true;
1943 }
1944
1945 /* Raise the IRQL to DISPATCH_LEVEL so we can't be rescheduled to another cpu.
1946 KeInsertQueueDpc must also be executed at IRQL >= DISPATCH_LEVEL. */
1947 KIRQL oldIrql;
1948 KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
1949
1950 if (g_pfnrtKeSetImportanceDpc)
1951 g_pfnrtKeSetImportanceDpc(&s_aPokeDpcs[idCpu], HighImportance);
1952 g_pfnrtKeSetTargetProcessorDpc(&s_aPokeDpcs[idCpu], (int)idCpu);
1953
1954 /* Assuming here that high importance DPCs will be delivered immediately; or at least an IPI will be sent immediately.
1955 Note! Not true on at least Vista & Windows 7 */
1956 BOOLEAN fRet = KeInsertQueueDpc(&s_aPokeDpcs[idCpu], 0, 0);
1957
1958 KeLowerIrql(oldIrql);
1959 return fRet == TRUE ? VINF_SUCCESS : VERR_ACCESS_DENIED /* already queued */;
1960}
1961
1962
1963RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
1964{
1965 if (!RTMpIsCpuOnline(idCpu))
1966 return !RTMpIsCpuPossible(idCpu)
1967 ? VERR_CPU_NOT_FOUND
1968 : VERR_CPU_OFFLINE;
1969 /* Calls rtMpPokeCpuUsingDpc, rtMpPokeCpuUsingHalRequestIpiW7Plus or rtMpPokeCpuUsingBroadcastIpi. */
1970 return g_pfnrtMpPokeCpuWorker(idCpu);
1971}
1972
1973
1974RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
1975{
1976 return false;
1977}
1978
注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette