diff --git a/lib/3rdparty/pybind11 b/lib/3rdparty/pybind11 index 8b03ffa7c0..01ab935612 160000 --- a/lib/3rdparty/pybind11 +++ b/lib/3rdparty/pybind11 @@ -1 +1 @@ -Subproject commit 8b03ffa7c06cd9c8a38297b1c8923695d1ff1b07 +Subproject commit 01ab935612a6800c4ad42957808d6cbd30047902 diff --git a/lib/vistle/core/shm_reference.h b/lib/vistle/core/shm_reference.h index 465c347e06..2c322c7160 100644 --- a/lib/vistle/core/shm_reference.h +++ b/lib/vistle/core/shm_reference.h @@ -254,7 +254,11 @@ class ShmArrayProxy { : m_data(arr ? arr->data() : nullptr) #endif , m_size(arr ? arr->size() : 0) +#ifdef NO_SHMEM + , m_handle(s_nullHandle) +#else , m_handle(arr ? arr->handle() : s_nullHandle) +#endif {} ShmArrayProxy &operator=(std::nullptr_t p) @@ -297,11 +301,12 @@ class ShmArrayProxy { #ifdef NO_SHMEM m_arr = &*ref; m_data = nullptr; + m_handle = s_nullHandle; #else m_data = ref->data(); + m_handle = ref->handle(); #endif m_size = ref->size(); - m_handle = ref->handle(); } else { #ifdef NO_SHMEM m_arr = nullptr; @@ -340,6 +345,7 @@ class ShmArrayProxy { } const vtkm::cont::ArrayHandle &handle() const { + updateFromHandle(); return m_handle; } @@ -361,11 +367,22 @@ class ShmArrayProxy { if (m_data) { return; } - m_data = m_arr->data(); + if (m_arr) { + // order is important: handle() may shrink_to_fit + m_handle = m_arr->handle(); + m_data = m_arr->data(); + } else { + m_handle = s_nullHandle; + m_data = nullptr; + } #endif } size_t m_size = 0; +#ifdef NO_SHMEM + mutable vtkm::cont::ArrayHandle m_handle; +#else vtkm::cont::ArrayHandle m_handle; +#endif static inline vtkm::cont::ArrayHandle s_nullHandle = vtkm::cont::make_ArrayHandle(static_cast(nullptr), 0, vtkm::CopyFlag::Off); }; diff --git a/lib/vistle/manager/clustermanager.cpp b/lib/vistle/manager/clustermanager.cpp index c14c0fdc07..4aa0f2897e 100644 --- a/lib/vistle/manager/clustermanager.cpp +++ b/lib/vistle/manager/clustermanager.cpp @@ -364,16 +364,16 @@ bool ClusterManager::checkBarrier(const message::uuid_t &uuid) const } } #ifdef BARRIER_DEBUG - CERR << "checkBarrier " << uuid << ": #local=" << numLocal << ", #reached=" << reachedSet.size() << std::endl; + CERR << "checkBarrier " << uuid << ": #local=" << numLocal << ", #reached=" << m_reachedSet.size() << std::endl; #endif - return reachedSet.size() == numLocal; + return m_reachedSet.size() == numLocal; } void ClusterManager::barrierReached(const message::uuid_t &uuid) { assert(m_barrierActive); m_comm.barrier(); - reachedSet.clear(); + m_reachedSet.clear(); CERR << "Barrier [" << uuid << "] reached" << std::endl; message::BarrierReached m(uuid); m.setDestId(message::Id::MasterHub); @@ -414,15 +414,15 @@ bool ClusterManager::dispatch(bool &received) const int modId = mod.id; // keep messages from modules that have already reached a barrier on hold - if (reachedSet.find(modId) != reachedSet.end()) + if (m_reachedSet.find(modId) != m_reachedSet.end()) continue; if (mod.hub == hubId()) { bool recv = false; message::Buffer buf; std::shared_ptr mq; - auto it = runningMap.find(modId); - if (it != runningMap.end()) { + auto it = m_runningMap.find(modId); + if (it != m_runningMap.end()) { it->second.update(); mq = it->second.recvQueue; } @@ -449,10 +449,10 @@ bool ClusterManager::dispatch(bool &received) } #else std::deque delayed; - for (auto &id_mod: runningMap) { + for (auto &id_mod: m_runningMap) { auto &id = id_mod.first; auto &mod = id_mod.second; - if (reachedSet.find(id) != reachedSet.end()) + if (m_reachedSet.find(id) != m_reachedSet.end()) continue; // process messages that have been delayed because of a previous barrier auto &incoming = mod.incomingMessages; @@ -471,18 +471,18 @@ bool ClusterManager::dispatch(bool &received) } while (!incoming.empty()) { int sender = incoming.front().buf.senderId(); - bool barrierReached = reachedSet.find(sender) != reachedSet.end(); + bool barrierReached = m_reachedSet.find(sender) != m_reachedSet.end(); if (!barrierReached) { if (!Communicator::the().handleMessage(incoming.front().buf, incoming.front().payload)) done = true; } - auto it = runningMap.find(sender); + auto it = m_runningMap.find(sender); if (barrierReached) { - if (it != runningMap.end()) { + if (it != m_runningMap.end()) { it->second.incomingMessages.emplace_back(incoming.front().buf, incoming.front().payload); } } else { - if (it != runningMap.end()) { + if (it != m_runningMap.end()) { it->second.update(); } } @@ -490,8 +490,8 @@ bool ClusterManager::dispatch(bool &received) } #endif - for (auto &mod: runningMap) { - bool barrierReached = reachedSet.find(mod.first) != reachedSet.end(); + for (auto &mod: m_runningMap) { + bool barrierReached = m_reachedSet.find(mod.first) != m_reachedSet.end(); if (!barrierReached) mod.second.update(); } @@ -542,7 +542,7 @@ bool ClusterManager::sendAllOthers(int excluded, const message::Message &message } // handle messages to modules - for (auto it = runningMap.begin(), next = it; it != runningMap.end(); it = next) { + for (auto it = m_runningMap.begin(), next = it; it != m_runningMap.end(); it = next) { // modules might be removed during message processing next = it; ++next; @@ -590,8 +590,8 @@ bool ClusterManager::sendMessage(const int moduleId, const message::Message &mes if (hub == hubId()) { //CERR << "local send to " << moduleId << ": " << buf << std::endl; if (destRank == -1 || destRank == getRank()) { - RunningMap::const_iterator it = runningMap.find(moduleId); - if (it == runningMap.end()) { + RunningMap::const_iterator it = m_runningMap.find(moduleId); + if (it == m_runningMap.end()) { CERR << "sendMessage: module " << moduleId << " not found" << std::endl; std::cerr << " message: " << buf << std::endl; return true; @@ -935,7 +935,7 @@ bool ClusterManager::handlePriv(const message::Spawn &spawn) int newId = spawn.spawnId(); std::string name(spawn.getName()); - Module &mod = runningMap[newId]; + Module &mod = m_runningMap[newId]; std::string smqName = message::MessageQueue::createName("send", newId, m_rank); std::string rmqName = message::MessageQueue::createName("recv", newId, m_rank); @@ -952,8 +952,8 @@ bool ClusterManager::handlePriv(const message::Spawn &spawn) mod.sendQueue->makeNonBlocking(); std::thread mt([this, newId, name, &mod]() { - std::string mname = "vistle:mq" + std::to_string(newId); - setThreadName(mname); + std::string tname = std::to_string(newId) + "mq:" + name; + setThreadName(tname); for (;;) { message::Buffer buf; @@ -1035,8 +1035,8 @@ bool ClusterManager::handlePriv(const message::Spawn &spawn) if (mod.newModule) { boost::mpi::communicator ncomm(m_comm, boost::mpi::comm_duplicate); std::thread t([newId, name, ncomm, &mod]() { - std::string mname = "vistle:" + name + ":" + std::to_string(newId); - setThreadName(mname); + std::string tname = std::to_string(newId) + "mn:" + name; + setThreadName(tname); //CERR << "thread for module " << name << ":" << newId << std::endl; mod.instance = mod.newModule(name, newId, ncomm); if (mod.instance) @@ -1048,9 +1048,9 @@ bool ClusterManager::handlePriv(const message::Spawn &spawn) prep.setAsPlugin(true); } else { CERR << "no newModule method for module " << name << std::endl; - auto it = runningMap.find(newId); - if (it != runningMap.end()) { - runningMap.erase(it); + auto it = m_runningMap.find(newId); + if (it != m_runningMap.end()) { + m_runningMap.erase(it); } // synthesize ModuleExit for module that has failed to start @@ -1178,9 +1178,9 @@ bool ClusterManager::handlePriv(const message::ModuleExit &moduleExit) if (moduleExit.isForwarded()) { sendAllOthers(mod, moduleExit, MessagePayload(), true); - RunningMap::iterator it = runningMap.find(mod); - if (it != runningMap.end()) { - runningMap.erase(it); + RunningMap::iterator it = m_runningMap.find(mod); + if (it != m_runningMap.end()) { + m_runningMap.erase(it); } else { //CERR << " Module [" << mod << "] not found in map" << std::endl; } @@ -1190,7 +1190,7 @@ bool ClusterManager::handlePriv(const message::ModuleExit &moduleExit) const bool local = isLocal(mod); if (local) { - if (runningMap.find(mod) == runningMap.end()) { + if (m_runningMap.find(mod) == m_runningMap.end()) { CERR << " Module [" << mod << "] quit, but not found in running map" << std::endl; return true; } @@ -1200,9 +1200,9 @@ bool ClusterManager::handlePriv(const message::ModuleExit &moduleExit) m_outputObjects.erase(port); } - ModuleSet::iterator it = reachedSet.find(mod); - if (it != reachedSet.end()) { - reachedSet.erase(it); + ModuleSet::iterator it = m_reachedSet.find(mod); + if (it != m_reachedSet.end()) { + m_reachedSet.erase(it); } else { if (m_barrierActive && checkBarrier(m_barrierUuid)) barrierReached(m_barrierUuid); @@ -1223,8 +1223,8 @@ bool ClusterManager::handlePriv(const message::ModuleExit &moduleExit) bool ClusterManager::handlePriv(const message::Execute &exec) { assert(exec.getModule() >= Id::ModuleBase); - RunningMap::iterator i = runningMap.find(exec.getModule()); - if (i == runningMap.end()) { + RunningMap::iterator i = m_runningMap.find(exec.getModule()); + if (i == m_runningMap.end()) { CERR << "did not find module to be executed: " << exec.getModule() << std::endl; return true; } @@ -1321,8 +1321,8 @@ bool ClusterManager::handlePriv(const message::Execute &exec) bool ClusterManager::handlePriv(const message::CancelExecute &cancel) { assert(cancel.getModule() >= Id::ModuleBase); - RunningMap::iterator i = runningMap.find(cancel.getModule()); - if (i == runningMap.end()) { + RunningMap::iterator i = m_runningMap.find(cancel.getModule()); + if (i == m_runningMap.end()) { CERR << "did not find module to cancel execution: " << cancel.getModule() << std::endl; return true; } @@ -1470,8 +1470,8 @@ bool ClusterManager::addObjectDestination(const message::AddObject &addObj, Obje if (!obj) { // block messages of receiving module until remote object is available assert(!isLocal(addObj.senderId())); - auto it = runningMap.find(destId); - if (it != runningMap.end()) { + auto it = m_runningMap.find(destId); + if (it != m_runningMap.end()) { Communicator::the().dataManager().requestObject( addObj, addObj.objectName(), [this, addObj, addObj2, broadcast](Object::const_ptr newobj) mutable { auto obj = addObj.getObject(); @@ -1630,9 +1630,9 @@ bool ClusterManager::handlePriv(const message::AddObjectCompleted &complete) bool ClusterManager::handlePriv(const message::ExecutionProgress &prog) { const bool localSender = idToHub(prog.senderId()) == hubId(); - RunningMap::iterator i = runningMap.find(prog.senderId()); + RunningMap::iterator i = m_runningMap.find(prog.senderId()); ClusterManager::Module *mod = nullptr; - if (i == runningMap.end()) { + if (i == m_runningMap.end()) { assert(localSender == false); } else { mod = &i->second; @@ -1816,8 +1816,8 @@ bool ClusterManager::handlePriv(const message::ExecutionProgress &prog) if (isLocal(destId)) { if (it->second.schedulingPolicy == message::SchedulingPolicy::LazyGang) { broadcast = true; - RunningMap::iterator i = runningMap.find(destId); - assert(i != runningMap.end()); + RunningMap::iterator i = m_runningMap.find(destId); + assert(i != m_runningMap.end()); auto &destMod = i->second; assert(destMod.prepared); @@ -1889,7 +1889,7 @@ bool ClusterManager::handlePriv(const message::Busy &busy) { if (getRank() == 0) { int id = busy.senderId(); - auto &mod = runningMap[id]; + auto &mod = m_runningMap[id]; if (mod.busyCount == 0) { message::Buffer buf(busy); buf.setDestId(Id::UI); @@ -1906,7 +1906,7 @@ bool ClusterManager::handlePriv(const message::Idle &idle) { if (getRank() == 0) { int id = idle.senderId(); - auto &mod = runningMap[id]; + auto &mod = m_runningMap[id]; --mod.busyCount; if (mod.busyCount == 0) { message::Buffer buf(idle); @@ -1931,9 +1931,9 @@ bool ClusterManager::handlePriv(const message::SetParameter &setParam) m_compressionSettingsValid = false; int sender = setParam.senderId(); int dest = setParam.destId(); - RunningMap::iterator i = runningMap.find(setParam.getModule()); + RunningMap::iterator i = m_runningMap.find(setParam.getModule()); Module *mod = nullptr; - if (i == runningMap.end()) { + if (i == m_runningMap.end()) { if (isLocal(setParam.getModule()) && setParam.getModule() != message::Id::Config) { CERR << "did not find module for SetParameter: " << setParam.getModule() << ": " << setParam << std::endl; } @@ -2034,12 +2034,12 @@ bool ClusterManager::handlePriv(const message::BarrierReached &barrReached) if (Id::isModule(barrReached.senderId())) { assert(isLocal(barrReached.senderId())); - reachedSet.insert(barrReached.senderId()); + m_reachedSet.insert(barrReached.senderId()); if (checkBarrier(m_barrierUuid)) { barrierReached(m_barrierUuid); #ifdef BARRIER_DEBUG } else { - CERR << "BARRIER: reached by " << reachedSet.size() << "/" << numRunning() << std::endl; + CERR << "BARRIER: reached by " << m_reachedSet.size() << "/" << numRunning() << std::endl; #endif } } else if (barrReached.senderId() == Id::MasterHub) { @@ -2202,7 +2202,7 @@ void ClusterManager::replayMessages() int ClusterManager::numRunning() const { int n = 0; - for (auto &m: runningMap) { + for (auto &m: m_runningMap) { int state = m_stateTracker.getModuleState(m.first); if ((state & StateObserver::Initialized) && /* !(state & StateObserver::Killed) && */ !(state & StateObserver::Quit)) @@ -2215,8 +2215,8 @@ bool ClusterManager::isReadyForExecute(int modId) const { //CERR << "checking whether " << modId << " can be executed: "; - auto i = runningMap.find(modId); - if (i == runningMap.end()) { + auto i = m_runningMap.find(modId); + if (i == m_runningMap.end()) { //CERR << "module " << modId << " not found" << std::endl; return false; } diff --git a/lib/vistle/manager/clustermanager.h b/lib/vistle/manager/clustermanager.h index 7a7c17ffd7..86eb225b53 100644 --- a/lib/vistle/manager/clustermanager.h +++ b/lib/vistle/manager/clustermanager.h @@ -199,7 +199,7 @@ class V_MANAGEREXPORT ClusterManager { bool haveDelayed() const; }; typedef std::unordered_map RunningMap; - RunningMap runningMap; + RunningMap m_runningMap; int numRunning() const; bool isReadyForExecute(int modId) const; int m_numExecuting = 0; @@ -211,7 +211,7 @@ class V_MANAGEREXPORT ClusterManager { message::uuid_t m_barrierUuid; int m_reachedBarriers; typedef std::set ModuleSet; - ModuleSet reachedSet; + ModuleSet m_reachedSet; std::vector m_numTransfering; long m_totalNumTransferring = 0; diff --git a/lib/vistle/module/module.cpp b/lib/vistle/module/module.cpp index 24b5f2dd40..bc81a7247e 100644 --- a/lib/vistle/module/module.cpp +++ b/lib/vistle/module/module.cpp @@ -2662,7 +2662,7 @@ bool Module::compute() m_tasks.push_back(task); std::unique_lock guard(task->m_mutex); - auto tname = name() + ":Block:" + std::to_string(m_tasks.size()); + auto tname = std::to_string(id()) + "b" + std::to_string(m_tasks.size()) + ":" + name(); task->m_future = std::async(std::launch::async, [this, tname, task] { PROF_FUNC(); setThreadName(tname); diff --git a/lib/vistle/module/reader.cpp b/lib/vistle/module/reader.cpp index 82afbec343..9713f1928f 100644 --- a/lib/vistle/module/reader.cpp +++ b/lib/vistle/module/reader.cpp @@ -155,7 +155,7 @@ bool Reader::readTimestep(std::shared_ptr &prev, const ReaderProperties & } m_tokens.emplace_back(token); prev = token; - auto tname = name() + ":Read:" + std::to_string(m_tokenCount); + auto tname = std::to_string(id()) + "r" + std::to_string(m_tokenCount) + ":" + name(); token->m_future = std::async(std::launch::async, [this, tname, token, timestep, p]() { setThreadName(tname); if (!read(*token, timestep, p)) { diff --git a/module/general/Transform/Transform.cpp b/module/general/Transform/Transform.cpp index 3acf827cb1..ada0fb2897 100644 --- a/module/general/Transform/Transform.cpp +++ b/module/general/Transform/Transform.cpp @@ -170,27 +170,32 @@ bool Transform::compute() transform *= rotMat; transform *= translateMat; - bool keep_original = p_keep_original->getValue(); int repetitions = p_repetitions->getValue(); AnimationMode animation = (AnimationMode)p_animation->getValue(); - int timestep = animation == Deanimate ? -1 : 0; - if (animation == TimestepAsRepetitionCount) { + int timestep = 0; + if (animation == Deanimate) { + timestep = -1; + } else if (animation == TimestepAsRepetitionCount) { timestep = split.timestep; repetitions = timestep; - } - - if (animation == TimestepAsPower) { + } else if (animation == TimestepAsPower) { timestep = split.timestep; - transform = pow(transform, timestep + 1); repetitions = 1; + if (timestep > 0) + transform = pow(transform, timestep); + else + transform = Matrix4::Identity(); } + bool keep_original = p_keep_original->getValue(); if (keep_original) { if (animation != Keep && animation != TimestepAsRepetitionCount) { Object::ptr outGeo; if (auto entry = m_cache.getOrLock(geo->getName(), outGeo)) { outGeo = geo->clone(); outGeo->setTimestep(timestep); + if (animation == Deanimate) + outGeo->setNumTimesteps(-1); updateMeta(outGeo); m_cache.storeAndUnlock(entry, outGeo); } @@ -202,8 +207,6 @@ bool Transform::compute() } else { addObject(data_out, outGeo); } - if (animation != Deanimate) - ++timestep; } else { Object::ptr nobj; if (auto entry = m_cache.getOrLock(obj->getName(), nobj)) { @@ -214,6 +217,7 @@ bool Transform::compute() addObject(data_out, nobj); } } + Matrix4 t = geo->getTransform(); for (int i = 0; i < repetitions; ++i) { t *= transform; @@ -222,11 +226,9 @@ bool Transform::compute() if (auto entry = m_cache.getOrLock(key, outGeo)) { outGeo = geo->clone(); outGeo->setTransform(t); - if (animation != Keep && animation != TimestepAsRepetitionCount) { - outGeo->setTimestep(timestep); - if (animation != Deanimate) - ++timestep; - } + outGeo->setTimestep(timestep); + if (animation == Deanimate) + outGeo->setNumTimesteps(-1); updateMeta(outGeo); m_cache.storeAndUnlock(entry, outGeo); } @@ -238,6 +240,10 @@ bool Transform::compute() } else { addObject(data_out, outGeo); } + if (animation != Keep && animation != TimestepAsRepetitionCount) { + if (animation != Deanimate) + ++timestep; + } } return true; diff --git a/module/map/Tracer/Particle.cpp b/module/map/Tracer/Particle.cpp index 24900cabfb..2040191730 100644 --- a/module/map/Tracer/Particle.cpp +++ b/module/map/Tracer/Particle.cpp @@ -95,7 +95,9 @@ void Particle::startTracing() assert(inGrid()); m_tracing = true; m_progressFuture = std::async(std::launch::async, [this]() -> bool { - setThreadName("Tracer:Particle:" + std::to_string(id())); + std::string tname = + std::to_string(m_global.module->id()) + "p" + std::to_string(id()) + ":" + m_global.module->name(); + setThreadName(tname); return trace(); }); } diff --git a/module/map/Tracer/Tracer.cpp b/module/map/Tracer/Tracer.cpp index 05f4ae1232..7e4c21e520 100644 --- a/module/map/Tracer/Tracer.cpp +++ b/module/map/Tracer/Tracer.cpp @@ -248,19 +248,20 @@ bool Tracer::compute() } if (useCelltree) { + std::string tname = std::to_string(id()) + "ct:" + name(); if (unstr) { - celltree[t + 1].emplace_back(std::async(std::launch::async, [unstr]() -> Celltree3::const_ptr { - setThreadName("Tracer:Celltree"); + celltree[t + 1].emplace_back(std::async(std::launch::async, [tname, unstr]() -> Celltree3::const_ptr { + setThreadName(tname); return unstr->getCelltree(); })); } else if (auto str = StructuredGrid::as(grid)) { - celltree[t + 1].emplace_back(std::async(std::launch::async, [str]() -> Celltree3::const_ptr { - setThreadName("Tracer:Celltree"); + celltree[t + 1].emplace_back(std::async(std::launch::async, [tname, str]() -> Celltree3::const_ptr { + setThreadName(tname); return str->getCelltree(); })); } else if (auto lg = LayerGrid::as(grid)) { - celltree[t + 1].emplace_back(std::async(std::launch::async, [lg]() -> Celltree3::const_ptr { - setThreadName("Tracer:Celltree"); + celltree[t + 1].emplace_back(std::async(std::launch::async, [tname, lg]() -> Celltree3::const_ptr { + setThreadName(tname); return lg->getCelltree(); })); } @@ -462,6 +463,7 @@ bool Tracer::reduce(int timestep) GlobalData global; + global.module = this; global.int_mode = (IntegrationMethod)getIntParameter("integration"); global.task_type = (TraceType)getIntParameter("taskType"); global.dt_step = getFloatParameter("dt_step"); diff --git a/module/map/Tracer/Tracer.h b/module/map/Tracer/Tracer.h index 7af843eab3..c97bb64975 100644 --- a/module/map/Tracer/Tracer.h +++ b/module/map/Tracer/Tracer.h @@ -19,6 +19,7 @@ DEFINE_ENUM_WITH_STRING_CONVERSIONS(TraceType, (Streamlines)(MovingPoints)(Pathlines)(Streaklines)) class BlockData; +class Tracer; class GlobalData { friend class Particle; @@ -64,6 +65,8 @@ class GlobalData { std::vector::ptr> blockField, cellField; std::vector::ptr> timeField, distField, stepWidthField; std::mutex mutex; + + Tracer *module = nullptr; }; class Tracer: public vistle::Module {