Geant4  10.03
G4MTRunManager.cc
Go to the documentation of this file.
1 //
2 // ********************************************************************
3 // * License and Disclaimer *
4 // * *
5 // * The Geant4 software is copyright of the Copyright Holders of *
6 // * the Geant4 Collaboration. It is provided under the terms and *
7 // * conditions of the Geant4 Software License, included in the file *
8 // * LICENSE and available at http://cern.ch/geant4/license . These *
9 // * include a list of copyright holders. *
10 // * *
11 // * Neither the authors of this software system, nor their employing *
12 // * institutes,nor the agencies providing financial support for this *
13 // * work make any representation or warranty, express or implied, *
14 // * regarding this software system or assume any liability for its *
15 // * use. Please see the license in the file LICENSE and URL above *
16 // * for the full disclaimer and the limitation of liability. *
17 // * *
18 // * This code implementation is the result of the scientific and *
19 // * technical work of the GEANT4 collaboration. *
20 // * By using, copying, modifying or distributing the software (or *
21 // * any work based on the software) you agree to acknowledge its *
22 // * use in resulting scientific publications, and indicate your *
23 // * acceptance of all terms of the Geant4 Software license. *
24 // ********************************************************************
25 //
26 //
27 
28 #include "G4MTRunManager.hh"
29 #include "G4MTRunManagerKernel.hh"
30 #include "G4Timer.hh"
31 #include "G4StateManager.hh"
32 #include "G4ScoringManager.hh"
37 #include "G4WorkerThread.hh"
38 #include "G4Run.hh"
39 #include "G4UImanager.hh"
40 #include "G4AutoLock.hh"
41 #include "G4WorkerRunManager.hh"
42 #include "G4UserRunAction.hh"
43 #include "G4ProductionCutsTable.hh"
44 #include "G4Timer.hh"
45 
50 
51 namespace {
52  G4Mutex cmdHandlingMutex = G4MUTEX_INITIALIZER;
53  G4Mutex scorerMergerMutex = G4MUTEX_INITIALIZER;
54  G4Mutex runMergerMutex = G4MUTEX_INITIALIZER;
55  G4Mutex setUpEventMutex = G4MUTEX_INITIALIZER;
56 }
57 
59 {
61  return fMasterRM;
65 }
66 
68 {
69  return fMasterRM->kernel;
70 }
71 
73 {
74  return fMasterRM->MTkernel;
75 }
76 
78  nworkers(2),forcedNwokers(-1),pinAffinity(0),
79  masterRNGEngine(0),
80  nextActionRequest(WorkerActionRequest::UNDEFINED),
81  eventModuloDef(0),eventModulo(1),
82  nSeedsUsed(0),nSeedsFilled(0),
83  nSeedsMax(10000),nSeedsPerEvent(2)
84 {
85  if ( fMasterRM )
86  {
87  G4Exception("G4MTRunManager::G4MTRunManager", "Run0110",FatalException,
88  "Another instance of a G4MTRunManager already exists.");
89  }
90  fMasterRM = this;
91  MTkernel = static_cast<G4MTRunManagerKernel*>(kernel);
92 #ifndef G4MULTITHREADED
94  msg << "Geant4 code is compiled without multi-threading support"
95  << "(-DG4MULTITHREADED is set to off).\n";
96  msg << "G4MTRunManager can only be used in multi-threaded applications.";
97  G4Exception("G4MTRunManager::G4MTRunManager","Run0111",FatalException,msg);
98 #endif
99 
100  G4int numberOfStaticAllocators = kernel->GetNumberOfStaticAllocators();
101  if(numberOfStaticAllocators>0)
102  {
104  msg1 << "There are " << numberOfStaticAllocators
105  << " static G4Allocator objects detected.\n"
106  << "In multi-threaded mode, all G4Allocator objects must be dynamicly instantiated.";
107  G4Exception("G4MTRunManager::G4MTRunManager","Run1035",FatalException,msg1);
108  }
111 
112  //Check if a default RandomNumberGenerator has been created by user,
113  // if not create default one
114  //Note this call forces creation of defaults if not already there
115  //G4Random::getTheEngine(); //User did not specify RNG, create defaults
116  //Now remember the master instance of the RNG Engine
117  masterRNGEngine = G4Random::getTheEngine();
118 
120  randDbl = new double[nSeedsPerEvent*nSeedsMax];
121 
122  char* env = getenv("G4FORCENUMBEROFTHREADS");
123  if(env)
124  {
125  G4String envS = env;
126  if(envS=="MAX"||envS=="max")
128  else
129  {
130  std::istringstream is(env);
131  G4int val = -1;
132  is >> val;
133  if(val>0)
134  { forcedNwokers = val; }
135  else
136  {
138  msg2 << "Environment variable G4FORCENUMBEROFTHREADS has an invalid value <"
139  << envS << ">. It has to be an integer or a word \"max\".\n"
140  << "G4FORCENUMBEROFTHREADS is ignored.";
141  G4Exception("G4MTRunManager::G4MTRunManager","Run1039",JustWarning,msg2);
142  }
143  }
144  if(forcedNwokers>0)
145  {
147  G4cout << "### Number of threads is forced to " << forcedNwokers
148  << " by Environment variable G4FORCENUMBEROFTHREADS." << G4endl;
149  }
150  }
151 }
152 
154 {
155  //TODO: Currently does not work due to concurrent deletion of something
156  // that is shared:
157  //G4ProcessTable::DeleteMessenger from ~G4RunManager
158  //G4cout<<"Destroy MTRunManager"<<G4endl;//ANDREA
160  delete [] randDbl;
161 }
162 
164 {
165  std::ostringstream os;
166  os << randomNumberStatusDir << "G4Master_"<<fn <<".rndm";
167  G4Random::saveEngineStatus(os.str().c_str());
168 }
169 
171 {
172  if ( threads.size() != 0 )
173  {
175  msg << "Number of threads cannot be changed at this moment \n"
176  << "(old threads are still alive). Method ignored.";
177  G4Exception("G4MTRunManager::SetNumberOfThreads(G4int)",
178  "Run0112", JustWarning, msg);
179  }
180  else if ( forcedNwokers > 0 )
181  {
183  msg << "Number of threads is forced to " << forcedNwokers
184  << " by G4FORCENUMBEROFTHREADS shell variable.\n"
185  << "Method ignored.";
186  G4Exception("G4MTRunManager::SetNumberOfThreads(G4int)",
187  "Run0113", JustWarning, msg);
188  }
189  else
190  {
191  nworkers = n;
192  }
193 }
194 
196 {
198 
199  // make sure all worker threads are set up.
200  BeamOn(0);
201  SetRunIDCounter(0);
203 }
204 
210 {
211  //Nothing to do
212 }
214 {
215  //Nothing to do
216 }
217 
219  G4AutoLock l(&cmdHandlingMutex);
220  uiCmdsForWorkers.clear();
221  std::vector<G4String>* cmdCopy = G4UImanager::GetUIpointer()->GetCommandStack();
222  for ( std::vector<G4String>::const_iterator it = cmdCopy->begin() ;
223  it != cmdCopy->end(); ++it )
224  uiCmdsForWorkers.push_back(*it);
225  cmdCopy->clear();
226  delete cmdCopy;
227 }
228 
229 std::vector<G4String> G4MTRunManager::GetCommandStack()
230 {
231  G4AutoLock l(&cmdHandlingMutex);
232  return uiCmdsForWorkers;
233 }
234 
236 {
237  //Now loop on requested number of workers
238  //This will also start the workers
239  //Currently we do not allow to change the
240  //number of threads: threads area created once
241  if ( threads.size() == 0 ) {
242  for ( G4int nw = 0 ; nw<nworkers; ++nw) {
243  //Create a new worker and remember it
244  G4WorkerThread* context = new G4WorkerThread;
245  context->SetNumberThreads(nworkers);
246  context->SetThreadId(nw);
248  threads.push_back(thread);
249  }
250  }
251  //Signal to threads they can start a new run
253 }
254 
255 
256 void G4MTRunManager::InitializeEventLoop(G4int n_event, const char* macroFile, G4int n_select)
257 {
259  numberOfEventToBeProcessed = n_event;
261 
262  if(!fakeRun)
263  {
264  nSeedsUsed = 0;
265  nSeedsFilled = 0;
266 
267  if(verboseLevel>0)
268  { timer->Start(); }
269 
270  n_select_msg = n_select;
271  if(macroFile!=0)
272  {
273  if(n_select_msg<0) n_select_msg = n_event;
274  msgText = "/control/execute ";
275  msgText += macroFile;
276  selectMacro = macroFile;
277  }
278  else
279  {
280  n_select_msg = -1;
281  selectMacro = "";
282  }
283 
284  //initialize seeds
285  //If user did not implement InitializeSeeds,
286  // use default: nSeedsPerEvent seeds per event
287  if( eventModuloDef > 0 )
288  {
291  {
293  if(eventModulo<1) eventModulo =1;
295  msgd << "Event modulo is reduced to " << eventModulo
296  << " to distribute events to all threads.";
297  G4Exception("G4MTRunManager::InitializeEventLoop()",
298  "Run10035", JustWarning, msgd);
299  }
300  }
301  else
302  {
303  eventModulo = int(std::sqrt(double(numberOfEventToBeProcessed/nworkers)));
304  if(eventModulo<1) eventModulo =1;
305  }
306  if ( InitializeSeeds(n_event) == false && n_event>0 )
307  {
310  {
311  case 0:
312  nSeedsFilled = n_event;
313  break;
314  case 1:
316  break;
317  case 2:
318  nSeedsFilled = n_event/eventModulo + 1;
319  break;
320  default:
322  msgd << "Parameter value <" << seedOncePerCommunication
323  << "> of seedOncePerCommunication is invalid. It is reset to 0." ;
324  G4Exception("G4MTRunManager::InitializeEventLoop()",
325  "Run10036", JustWarning, msgd);
326  seedOncePerCommunication = 0;
327  nSeedsFilled = n_event;
328  }
329 
330  // Generates up to nSeedsMax seed pairs only.
333  helper->Fill(randDbl,nSeedsFilled,n_event,nSeedsPerEvent);
334  }
335  }
336 
337  //Now initialize workers. Check if user defined a WorkerThreadInitialization
340 
341  //Prepare UI commands for threads
343 
344  //Start worker threads
346 
347  // We need a barrier here. Wait for workers to start event loop.
348  //This will return only when all workers have started processing events.
350 }
351 
353 {
355  G4int nFill = 0;
357  {
358  case 0:
360  break;
361  case 1:
362  nFill = nworkers - nSeedsFilled;
363  break;
364  case 2:
365  default:
366  nFill = (numberOfEventToBeProcessed - nSeedsFilled*eventModulo)/eventModulo + 1;
367  }
368  // Generates up to nSeedsMax seed pairs only.
369  if(nFill>nSeedsMax) nFill=nSeedsMax;
370  masterRNGEngine->flatArray(nSeedsPerEvent*nFill,randDbl);
371  helper->Refill(randDbl,nFill);
372  nSeedsFilled += nFill;
373 //G4cout<<"helper->Refill() for "<<nFill<<" events."<<G4endl;
374 }
375 
377 {
378  //Wait for all worker threads to have finished the run
379  //i.e. wait for them to return from RunTermination()
380  //This guarantee that userrunaction for workers has been called
381 
382  // Wait now for all threads to finish event-loop
384  //Now call base-class methof
387 }
388 
390 {
392  //Call base class stuff...
394 
395  masterWorlds.clear();
397  std::vector<G4VPhysicalVolume*>::iterator itrW
399  for(size_t iWorld=0;iWorld<nWorlds;iWorld++)
400  {
401  addWorld(iWorld,*itrW);
402  itrW++;
403  }
404 }
405 
407 {
408  userWorkerInitialization = userInit;
409 }
410 
412 {
414 }
415 
417 {
418  userActionInitialization = userInit;
420 }
421 
423 {
425  //Needed for MT, to be moved in kernel
426 }
427 
429 {
431 }
432 
434 {
435  G4RunManager::SetUserAction(userAction);
436  userAction->SetMaster();
437 }
438 
440 {
441  G4Exception("G4MTRunManager::SetUserAction()", "Run0123", FatalException,
442  "For multi-threaded version, define G4VUserPrimaryGeneratorAction in G4VUserActionInitialization.");
443 }
444 
446 {
447  G4Exception("G4MTRunManager::SetUserAction()", "Run0124", FatalException,
448  "For multi-threaded version, define G4UserEventAction in G4VUserActionInitialization.");
449 }
450 
452 {
453  G4Exception("G4MTRunManager::SetUserAction()", "Run0125", FatalException,
454  "For multi-threaded version, define G4UserStackingAction in G4VUserActionInitialization.");
455 }
456 
458 {
459  G4Exception("G4MTRunManager::SetUserAction()", "Run0126", FatalException,
460  "For multi-threaded version, define G4UserTrackingAction in G4VUserActionInitialization.");
461 }
462 
464 {
465  G4Exception("G4MTRunManager::SetUserAction()", "Run0127", FatalException,
466  "For multi-threaded version, define G4UserSteppingAction in G4VUserActionInitialization.");
467 }
468 
469 void G4MTRunManager::MergeScores(const G4ScoringManager* localScoringManager)
470 {
471  G4AutoLock l(&scorerMergerMutex);
472  if(masterScM) masterScM->Merge(localScoringManager);
473 }
474 
475 void G4MTRunManager::MergeRun(const G4Run* localRun)
476 {
477  G4AutoLock l(&runMergerMutex);
478  if(currentRun) currentRun->Merge(localRun);
479 }
480 
481 G4bool G4MTRunManager::SetUpAnEvent(G4Event* evt,long& s1,long& s2,long& s3,G4bool reseedRequired)
482 {
483  G4AutoLock l(&setUpEventMutex);
485  {
487  if(reseedRequired)
488  {
490  G4int idx_rndm = nSeedsPerEvent*nSeedsUsed;
491  s1 = helper->GetSeed(idx_rndm);
492  s2 = helper->GetSeed(idx_rndm+1);
493  if(nSeedsPerEvent==3) s3 = helper->GetSeed(idx_rndm+2);
494  nSeedsUsed++;
495  if(nSeedsUsed==nSeedsFilled) RefillSeeds();
496  }
498  return true;
499  }
500  return false;
501 }
502 
504 {
505  G4AutoLock l(&setUpEventMutex);
507  {
508  G4int nev = eventModulo;
512  if(reseedRequired)
513  {
515  G4int nevRnd = nev;
516  if(seedOncePerCommunication>0) nevRnd = 1;
517  for(int i=0;i<nevRnd;i++)
518  {
519  seedsQueue->push(helper->GetSeed(nSeedsPerEvent*nSeedsUsed));
520  seedsQueue->push(helper->GetSeed(nSeedsPerEvent*nSeedsUsed+1));
521  if(nSeedsPerEvent==3)
522  seedsQueue->push(helper->GetSeed(nSeedsPerEvent*nSeedsUsed+2));
523  nSeedsUsed++;
524  if(nSeedsUsed==nSeedsFilled) RefillSeeds();
525  }
526  }
527  numberOfEventProcessed += nev;
528  return nev;
529  }
530  return 0;
531 }
532 
534 {
535  //Force workers to execute (if any) all UI commands left in the stack
537  //Ask workers to exit
539  //Now join threads.
540 #ifdef G4MULTITHREADED //protect here to prevent warning in compilation
541  while ( ! threads.empty() )
542  {
543  G4Thread* t = * ( threads.begin() );
544  threads.pop_front();
546  //G4THREADJOIN(*t);
547  delete t;
548  }
549 #endif
550  threads.clear();
551 }
552 
554 {
555  // This method is valid only for GeomClosed or EventProc state
556  G4ApplicationState currentState =
558  if(currentState==G4State_GeomClosed || currentState==G4State_EventProc)
559  {
560  runAborted = true;
561  MTkernel->BroadcastAbortRun(softAbort);
562  }
563  else
564  {
565  G4cerr << "Run is not in progress. AbortRun() ignored." << G4endl;
566  }
567 }
568 
570 {
571  // nothing to do in the master thread
572 }
573 
578 }
579 
582 }
583 
588 }
589 
592 }
593 
596  //nextActionRequest is a shared resource, but there is no
597  //data-race thanks to the barrier: all threads are waiting
598  nextActionRequest = newRequest;
600 }
601 
604  return nextActionRequest;
605 }
606 
612 }
613 
616 }
618 {
619  if ( n == 0 )
620  {
621  G4Exception("G4MTRunManager::SetPinAffinity",
622  "Run0114",FatalException,
623  "Pin affinity must be >0 or <0.");
624  }
625  pinAffinity = n;
626  return;
627 }
G4Timer * timer
virtual void InitializeEventLoop(G4int n_event, const char *macroFile=0, G4int n_select=-1)
virtual void TerminateEventLoop()
virtual G4int SetUpNEvents(G4Event *, G4SeedsQueue *seedsQueue, G4bool reseedRequired=true)
std::vector< G4String > GetCommandStack()
virtual void PrepareCommandsStack()
virtual void SetUserInitialization(G4VUserDetectorConstruction *userInit)
std::queue< G4long > G4SeedsQueue
Definition: G4RNGHelper.hh:132
virtual void Merge(const G4Run *)
Definition: G4Run.cc:54
std::map< G4int, G4VPhysicalVolume * > masterWorlds_t
static G4int seedOncePerCommunication
G4int n_select_msg
std::vector< G4String > uiCmdsForWorkers
G4int numberOfEventProcessed
static void addWorld(G4int counter, G4VPhysicalVolume *w)
WorkerActionRequest nextActionRequest
virtual void RunTermination()
std::ostringstream G4ExceptionDescription
Definition: globals.hh:76
virtual void ThisWorkerEndEventLoop()
void SetNumberOfThreads(G4int n)
std::vector< G4VPhysicalVolume * >::iterator GetWorldsIterator()
G4VUserActionInitialization * userActionInitialization
G4UserWorkerThreadInitialization * userWorkerThreadInitialization
virtual G4bool InitializeSeeds(G4int)
virtual void ConstructScoringWorlds()
virtual void BeamOn(G4int n_event, const char *macroFile=0, G4int n_select=-1)
virtual void RunTermination()
virtual const T GetSeed(const G4int &sdId)
Definition: G4RNGHelper.hh:62
void SetActiveThreads(unsigned int val)
Definition: G4MTBarrier.hh:142
virtual void WaitForReadyWorkers()
Definition: G4MTBarrier.cc:104
void SetThreadId(G4int threadId)
G4String msgText
int G4int
Definition: G4Types.hh:78
G4bool runAborted
std::vector< G4String > * GetCommandStack()
Definition: G4UImanager.cc:709
virtual void SetUserAction(G4UserRunAction *userAction)
#define G4MUTEX_INITIALIZER
Definition: G4Threading.hh:175
G4int G4Thread
Definition: G4Threading.hh:174
void Refill(G4double *dbl, G4int nev)
Definition: G4RNGHelper.hh:92
void ThisWorkerReady()
Definition: G4MTBarrier.cc:51
G4MTBarrier beginOfEventLoopBarrier
virtual void WaitForEndEventLoopWorkers()
G4MTRunManagerKernel * MTkernel
virtual void TerminateOneEvent()
static G4UImanager * GetUIpointer()
Definition: G4UImanager.cc:58
void ResetCounter()
Definition: G4MTBarrier.cc:111
virtual void TerminateWorkers()
static G4StateManager * GetStateManager()
static G4RunManagerKernel * GetMasterRunManagerKernel()
G4ThreadsList threads
void Fill(G4double *dbl, G4int nev, G4int nev_tot, G4int nrpe)
Definition: G4RNGHelper.hh:81
G4GLOB_DLL std::ostream G4cout
void MergeRun(const G4Run *localRun)
static masterWorlds_t masterWorlds
static G4ScoringManager * GetScoringManagerIfExist()
static G4ScoringManager * masterScM
G4UserWorkerInitialization * userWorkerInitialization
G4String randomNumberStatusDir
G4MTBarrier processUIBarrier
virtual void ProcessOneEvent(G4int i_event)
bool G4bool
Definition: G4Types.hh:79
virtual void ConstructScoringWorlds()
virtual void WaitForReadyWorkers()
static G4MTRunManager * GetMasterRunManager()
G4int G4GetNumberOfCores()
Definition: G4Threading.cc:143
void SetPinAffinity(G4int n=1)
virtual WorkerActionRequest ThisWorkerWaitForNextAction()
virtual void ThisWorkerProcessCommandsStackDone()
Definition: G4Run.hh:46
virtual void SetUserInitialization(G4VUserPhysicsList *userPL)
virtual void AbortRun(G4bool softAbort=false)
virtual void RequestWorkersProcessCommandsStack()
G4ApplicationState GetCurrentState() const
virtual void NewActionRequest(WorkerActionRequest newRequest)
const G4int n
virtual void SetMaster(G4bool val=true)
void ReleaseBarrier()
Definition: G4MTBarrier.cc:97
void Merge(const G4ScoringManager *scMan)
void G4Exception(const char *originOfException, const char *exceptionCode, G4ExceptionSeverity severity, const char *comments)
Definition: G4Exception.cc:41
static G4TransportationManager * GetTransportationManager()
G4int G4Mutex
Definition: G4Threading.hh:173
virtual void Initialize()
G4String selectMacro
G4MTBarrier endOfEventLoopBarrier
virtual void ThisWorkerReady()
virtual void AbortEvent()
void SetRunIDCounter(G4int i)
virtual G4bool SetUpAnEvent(G4Event *, long &s1, long &s2, long &s3, G4bool reseedRequired=true)
virtual G4Thread * CreateAndStartWorker(G4WorkerThread *workerThreadContext)
virtual ~G4MTRunManager()
void Wait()
Definition: G4MTBarrier.cc:73
static G4TemplateRNGHelper< T > * GetInstance()
Definition: G4RNGHelper.cc:37
virtual void Initialize()
size_t GetNoWorlds() const
G4int numberOfEventToBeProcessed
G4Run * currentRun
#define G4endl
Definition: G4ios.hh:61
G4int GetNumberOfStaticAllocators() const
G4RunManagerKernel * kernel
void Start()
void BroadcastAbortRun(G4bool softAbort)
static G4MTRunManagerKernel * GetMTMasterRunManagerKernel()
G4MTBarrier nextActionRequestBarrier
size_t GetNumberActiveThreads() const
void SetNumberThreads(G4int numnberThreads)
void MergeScores(const G4ScoringManager *localScoringManager)
void SetEventID(G4int i)
Definition: G4Event.hh:115
CLHEP::HepRandomEngine * masterRNGEngine
void SetMasterUIManager(G4bool val)
Definition: G4UImanager.hh:258
G4ApplicationState
virtual void StoreRNGStatus(const G4String &filenamePrefix)
G4int verboseLevel
G4GLOB_DLL std::ostream G4cerr
virtual void SetUserAction(G4UserRunAction *userAction)
virtual void CreateAndStartWorkers()
static G4MTRunManager * fMasterRM