35 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
36 ownsBuffer(false),scoringManager(nullptr),commSize(0),
43 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
45 scoringManager(mgr), commSize(0), destinationRank(destination),
54 #define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
88 DMSG(0,
"G4MPIscorerMerger::Merge called");
89 const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
90 commSize = MPI::COMM_WORLD.Get_size();
92 DMSG(1,
"Comm world size is 1, nothing to do");
95 comm = MPI::COMM_WORLD.Dup();
114 const G4double sttime = MPI::Wtime();
117 typedef std::function<void(unsigned int)> handler_t;
118 using std::placeholders::_1;
121 std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier,&
comm);
140 const G4double elapsed = MPI::Wtime() - sttime;
145 G4cout<<
"G4MPIscorerMerger::Merge() -data transfer performances: "
146 <<double(total)/1000./elapsed<<
" kB/s"
147 <<
" (Total Data Transfer= "<<double(total)/1000.<<
" kB in "
148 <<elapsed<<
" s)."<<
G4endl;
165 DMSG(0,
"G4MPIscorerMerger::Merge done.");
169 DMSG(1,
"Receiving scorers");
171 DMSG(2,
"Receiving from: "<<source);
174 const G4int newbuffsize = status.Get_count(MPI::PACKED);
175 DMSG(2,
"Preparing to receive buffer of size: "<<newbuffsize);
178 DMSG(3,
"New larger buffer expected, resize");
182 buffer =
new char[newbuffsize];
185 std::fill( buffer , buffer + newbuffsize , 0 );
189 comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
193 DMSG(1,
"Receiving of comamnd line scorers done");
197 DMSG(1,
"Sending scorers "<<
this);
204 buffer =
new char[newbuffsize];
207 std::fill( buffer , buffer+newbuffsize,0);
220 DMSG(1,
"Sending done");
226 G4Exception(
"G4MPIscorerMerger::Pack(const G4ScoringManager*)",
228 "Call SetOututBuffer before trying to pack");
233 MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
237 for (
size_t i = 0; i <numMeshes; ++i)
239 MPI_Pack(&i,1,MPI::UNSIGNED,
249 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
251 "Call SetOututBuffer before trying to un-pack");
256 &numMeshes,1,MPI::UNSIGNED,
comm);
259 msg <<
"Number of meshes to unpack ("<<numMeshes;
260 msg <<
") does not correspond to expected number ("<<sm->
GetNumberOfMesh();
262 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
268 for (
size_t i = 0 ; i < numMeshes ; ++i ) {
270 &meshid,1,MPI::UNSIGNED,
comm);
273 msg<<
"Cannot unpack: expecting mesh "<<i<<
" and found "<<meshid;
274 msg<<
" during unpack.";
275 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
285 assert(mesh!=
nullptr);
288 DMSG(3,
"Packing mesh: "<<mesh);
291 size_t nummaps = map.size();
292 MPI_Pack(&nummaps,1,MPI::UNSIGNED,
295 for (
const auto&
ele: map ) {
297 size_t ss = name.size();
298 MPI_Pack(&ss,1,MPI::UNSIGNED,
301 #ifdef G4MPI_USE_MPI_PACK_NOT_CONST
302 char*
nn =
new char[name.length()];
303 std::copy(name.begin(),name.end(),
nn);
305 const char* nn = name.c_str();
307 MPI_Pack(nn,ss,MPI::CHAR,
311 #ifdef G4MPI_USE_MPI_PACK_NOT_CONST
320 assert(inmesh!=
nullptr);
321 DMSG(3,
"Preparing to unpack a mesh and merge into: "<<inmesh);
325 &nummaps,1,MPI::UNSIGNED,
comm);
326 for (
size_t i = 0 ; i < nummaps ; ++i ) {
329 &nameSize,1,MPI::UNSIGNED,
comm);
332 char*
name =
new char[nameSize+1];
333 std::fill(name,name+nameSize+1,0);
335 name,nameSize,MPI::CHAR,
comm);
336 const G4String colname(name,nameSize);
351 DMSG(3,
"Packing hitmap: "<<sm<<
" with: "<<sm->
GetSize()<<
" elements.");
353 MPI_Pack(&numEl,1,MPI::UNSIGNED,
356 const auto& theMap = *sm->
GetMap();
357 std::vector<G4int> ids;
358 std::vector<G4double> vals;
359 std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
360 [](decltype(*theMap.begin())& e){
return e.first;});
361 std::transform(theMap.begin(),theMap.end(),std::back_inserter(vals),
362 [](decltype(*theMap.begin())& e){
return *e.second;});
363 assert(ids.size()==vals.size()&&ids.size()==numEl);
364 MPI_Pack(ids.data(),ids.size(),
MPI::INT,
376 DMSG(3,
"Preparing to unpack a hit map for: "<<detName<<
","<<colName);
379 &numEl,1,MPI::UNSIGNED,
comm);
387 for (
unsigned int i = 0; i<numEl;++i) result->
set(ids[i],vals[i]);
395 DMSG(3,
"Calculating dimension of data to send");
396 if ( sm ==
nullptr )
return 0;
402 G4int size =
sizeof(
unsigned int);
406 size +=
sizeof(
unsigned int);
414 DMSG(3,
"Calculating size for mesh: "<<mesh);
416 G4int size =
sizeof(
unsigned int);
418 for (
const auto&
ele : map ) {
420 size +=
sizeof(
unsigned int);
422 size +=
sizeof(char)*name.size();
425 DMSG(3,
"mesh "<<mesh<<
" size: "<<size);
431 G4int size =
sizeof(
unsigned int);
432 size +=
sizeof(
G4int)*numEls;
434 DMSG(3,
"HitMap "<<map<<
" size: "<<size<<
" in "<<numEls<<
" elements.");
const G4String & GetWorldName() const
std::ostringstream G4ExceptionDescription
void Pack(const G4ScoringManager *)
Pack all meshes into buffer.
std::map< G4String, G4THitsMap< G4double > * > MeshScoreMap
G4ScoringManager * scoringManager
unsigned int destinationRank
void UnPackAndMerge(const G4ScoringManager *)
void Accumulate(G4THitsMap< G4double > *map)
void Merge(std::function< void(unsigned int)> senderF, std::function< void(unsigned int)> receiverF, std::function< void(void)> barrierF, unsigned int commSize, unsigned int myrank)
void Send(const unsigned int destination)
void Receive(const unsigned int source)
G4GLOB_DLL std::ostream G4cout
HitMap * UnPackHitMap(const G4String &detName, const G4String &colName)
G4THitsMap< G4double > HitMap
G4int CalculatePackSize(const G4ScoringManager *) const
virtual size_t GetSize() const
virtual ~G4MPIscorerMerger()
void G4Exception(const char *originOfException, const char *exceptionCode, G4ExceptionSeverity severity, const char *comments)
G4double total(Particle const *const p1, Particle const *const p2)
G4int outputBufferPosition
static const G4double ele
MeshScoreMap GetScoreMap() const
std::map< G4int, T * > * GetMap() const
static G4String Status(G4StepStatus stps)
size_t GetNumberOfMesh() const
void SetupOutputBuffer(char *buff, G4int size, G4int position)
G4VScoringMesh * GetMesh(G4int i) const
G4int set(const G4int &key, T *&aHit) const