Memsync

CMakeLists.txt

@@ -102,6 +102,22 @@ message(STATUS ">> Build with PINT_DEBUG : " ${BUILD_WITH_PINT_DEBUG})
 option(BUILD_WITH_ADIOS2 "Enable output in the Adios2 format." OFF)
 message(STATUS ">> Build with Adios2: " ${BUILD_WITH_ADIOS2})
 
+# Profiling
+if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR (CMAKE_BUILD_TYPE MATCHES "^Debug"))
+  # Support source code view in ncu-ui
+  add_compile_options(
+    $<$<COMPILE_LANGUAGE:CUDA>:-lineinfo>
+  )
+endif()
+
+# Force -O3 optimization in RelWithDebInfo builds
+# (Avoids multiple optimization levels in the compiler flags)
+if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+  string(REPLACE "-O2" "-O3"
+         CMAKE_CXX_FLAGS_RELWITHDEBINFO
+         "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+endif()
+
 #
 # Find dependencies
 #
@@ -308,7 +324,7 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "DebugOptimized")
     target_compile_options(simplemd_interface INTERFACE --fmad=false)
   endif()
 elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-  target_compile_options(simplemd_interface INTERFACE -O3 -g3)
+  target_compile_options(simplemd_interface INTERFACE -g3)
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
   if(KOKKOS_TARGET STREQUAL "CUDA")
     target_compile_options(simplemd_interface INTERFACE -O3)
@@ -525,7 +541,7 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "DebugOptimized")
   target_compile_definitions(mamico INTERFACE MDCoupledError)
   target_compile_options(mamico INTERFACE -O3 -g3)
 elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-  target_compile_options(mamico INTERFACE -O3 -g3)
+  target_compile_options(mamico INTERFACE -g3)
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
   if(KOKKOS_TARGET STREQUAL "CUDA")
     target_compile_options(mamico INTERFACE -O3)

simplemd/LinkedCell.h

@@ -17,8 +17,9 @@ class LinkedCell;
 class simplemd::LinkedCell {
 public:
   KOKKOS_FUNCTION LinkedCell() : _moleculeData(nullptr), _linkedCellNumMolecules(nullptr), _cellIndex(0), _isGhostCell(false) {}
-  KOKKOS_FUNCTION LinkedCell(const Kokkos::View<Molecule**, Kokkos::LayoutRight, Kokkos::SharedSpace>* moleculeData,
-                             const Kokkos::View<size_t*, Kokkos::LayoutRight, Kokkos::SharedSpace>* nMolecules, unsigned int cellIndex, bool isGhostCell)
+
+  KOKKOS_FUNCTION LinkedCell(const void* moleculeData,
+                             const void* nMolecules, unsigned int cellIndex, bool isGhostCell)
       : _moleculeData(moleculeData), _linkedCellNumMolecules(nMolecules), _cellIndex(cellIndex), _isGhostCell(isGhostCell) {}
 
   class Iterator {
@@ -75,27 +76,55 @@ class simplemd::LinkedCell {
     *getMolecule(moleculeIdx) = *getMolecule(numMolecules() - 1);
     changeMoleculeCount(-1);
   }
-  KOKKOS_FUNCTION void clear() { (*_linkedCellNumMolecules)(_cellIndex) = 0; }
+  KOKKOS_FUNCTION void clear() { 
+    KOKKOS_IF_ON_HOST((
+      (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>::host_mirror_type*>(_linkedCellNumMolecules))(_cellIndex) = 0; 
+    ))
+    KOKKOS_IF_ON_DEVICE((
+      (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>*>(_linkedCellNumMolecules))(_cellIndex) = 0;
+    ))
+  }
 
   std::string to_string() const {
     std::stringstream to_ret;
     to_ret << " numMol: " << numMolecules() << std::endl;
     return to_ret.str();
   }
 
-  KOKKOS_INLINE_FUNCTION unsigned int numMolecules() const { return (*_linkedCellNumMolecules)(_cellIndex); }
+  KOKKOS_INLINE_FUNCTION unsigned int numMolecules() const { 
+    KOKKOS_IF_ON_HOST((
+      return (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>::host_mirror_type*>(_linkedCellNumMolecules))(_cellIndex); 
+    ))
+    KOKKOS_IF_ON_DEVICE((
+      return (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>*>(_linkedCellNumMolecules))(_cellIndex); 
+    ))
+  }
 
   KOKKOS_INLINE_FUNCTION bool isGhostCell() const { return _isGhostCell; }
 
   KOKKOS_INLINE_FUNCTION size_t getIndex() const { return _cellIndex; }
 
 private:
-  KOKKOS_INLINE_FUNCTION void changeMoleculeCount(int by) { (*_linkedCellNumMolecules)(_cellIndex) += by; }
+  KOKKOS_INLINE_FUNCTION void changeMoleculeCount(int by) { 
+    KOKKOS_IF_ON_HOST((
+      (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>::host_mirror_type*>(_linkedCellNumMolecules))(_cellIndex) += by; 
+    ))
+    KOKKOS_IF_ON_DEVICE((
+      (*static_cast<const Kokkos::View<size_t*, Kokkos::LayoutRight>*>(_linkedCellNumMolecules))(_cellIndex) += by; 
+    ))
+  }
 
-  KOKKOS_INLINE_FUNCTION Molecule* getMolecule(unsigned int moleculeIndex) const { return &(*_moleculeData)(_cellIndex, moleculeIndex); }
+  KOKKOS_INLINE_FUNCTION Molecule* getMolecule(unsigned int moleculeIndex) const { 
+    KOKKOS_IF_ON_HOST((
+      return &(*static_cast<const Kokkos::View<Molecule**, Kokkos::LayoutRight>::host_mirror_type*>(_moleculeData))(_cellIndex, moleculeIndex);
+    ))
+    KOKKOS_IF_ON_DEVICE((
+      return &(*static_cast<const Kokkos::View<Molecule**, Kokkos::LayoutRight>*>(_moleculeData))(_cellIndex, moleculeIndex);
+    ))
+  }
 
-  const Kokkos::View<Molecule**, Kokkos::LayoutRight, Kokkos::SharedSpace>* _moleculeData;
-  const Kokkos::View<size_t*, Kokkos::LayoutRight, Kokkos::SharedSpace>* _linkedCellNumMolecules;
+  const void* _moleculeData;
+  const void* _linkedCellNumMolecules;
   const unsigned int _cellIndex;
   const bool _isGhostCell;
 };

simplemd/MolecularDynamicsSimulation.cpp

@@ -531,7 +531,8 @@ void simplemd::MolecularDynamicsSimulation::simulateOneTimestep(const unsigned i
   //     buffers to arrive, then compute forces near boundaries, all within
   //     putBoundaryParticlesToInnerCellsFillBoundaryCellsAndOverlapWithForceComputations.
   if (!_configuration.getSimulationConfiguration().useOverlappingCommunicationWithForceComputation()) {
-    _boundaryTreatment->putBoundaryParticlesToInnerCellsAndFillBoundaryCells(_localBoundary, *_parallelTopologyService);
+    // TODO refactor boundaryTreatment
+    //_boundaryTreatment->putBoundaryParticlesToInnerCellsAndFillBoundaryCells(_localBoundary, *_parallelTopologyService);
     // compute forces between molecules.
 #if defined(KOKKOS_TARGET_CUDA)
     _moleculeService->getContainer().iterateMoleculesWithCell(*_lennardJonesForce);
@@ -548,7 +549,8 @@ void simplemd::MolecularDynamicsSimulation::simulateOneTimestep(const unsigned i
 
   evaluateStatistics(t);
 
-  _boundaryTreatment->emptyGhostBoundaryCells();
+  // TODO refactor boundaryTreatment
+  //_boundaryTreatment->emptyGhostBoundaryCells();
 
   // plot VTK output
   if ((_configuration.getVTKConfiguration().getWriteEveryTimestep() > 0) && (t % _configuration.getVTKConfiguration().getWriteEveryTimestep() == 0)) {

simplemd/Molecule.h

@@ -36,27 +36,27 @@ class simplemd::Molecule {
 
   /** get/set ID */
   KOKKOS_FUNCTION const unsigned int& getID() const { return _id; }
-  void setID(const unsigned int& id) { _id = id; }
+  KOKKOS_FUNCTION void setID(const unsigned int& id) { _id = id; }
 
   /** get/ set position */
   KOKKOS_FUNCTION tarch::la::Vector<MD_DIM, double>& getPosition() { return _position; }
   KOKKOS_FUNCTION const tarch::la::Vector<MD_DIM, double>& getConstPosition() const { return _position; }
-  void setPosition(const tarch::la::Vector<MD_DIM, double>& position) { _position = position; }
+  KOKKOS_FUNCTION void setPosition(const tarch::la::Vector<MD_DIM, double>& position) { _position = position; }
 
   /** get/ set velocity */
-  tarch::la::Vector<MD_DIM, double>& getVelocity() { return _velocity; }
+  KOKKOS_FUNCTION tarch::la::Vector<MD_DIM, double>& getVelocity() { return _velocity; }
   KOKKOS_FUNCTION const tarch::la::Vector<MD_DIM, double>& getConstVelocity() const { return _velocity; }
-  void setVelocity(const tarch::la::Vector<MD_DIM, double>& velocity) { _velocity = velocity; }
+  KOKKOS_FUNCTION void setVelocity(const tarch::la::Vector<MD_DIM, double>& velocity) { _velocity = velocity; }
 
   /** get/ set force */
   KOKKOS_FUNCTION tarch::la::Vector<MD_DIM, double>& getForce() { return _force; }
   KOKKOS_FUNCTION const tarch::la::Vector<MD_DIM, double>& getConstForce() const { return _force; }
-  void setForce(const tarch::la::Vector<MD_DIM, double>& force) { _force = force; }
+  KOKKOS_FUNCTION void setForce(const tarch::la::Vector<MD_DIM, double>& force) { _force = force; }
 
   /** get/ set force of last timestep */
-  tarch::la::Vector<MD_DIM, double>& getForceOld() { return _forceOld; }
-  const tarch::la::Vector<MD_DIM, double>& getConstForceOld() const { return _forceOld; }
-  void setForceOld(const tarch::la::Vector<MD_DIM, double>& force) { _forceOld = force; }
+  KOKKOS_FUNCTION tarch::la::Vector<MD_DIM, double>& getForceOld() { return _forceOld; }
+  KOKKOS_FUNCTION const tarch::la::Vector<MD_DIM, double>& getConstForceOld() const { return _forceOld; }
+  KOKKOS_FUNCTION void setForceOld(const tarch::la::Vector<MD_DIM, double>& force) { _forceOld = force; }
 
   KOKKOS_FUNCTION double& getPotentialEnergy() { return _potentialEnergy; }
   KOKKOS_FUNCTION const double& getConstPotentialEnergy() const { return _potentialEnergy; }