remhos changes

experiments/remhos/experiment.py

@@ -17,7 +17,7 @@ class Remhos(
         ProgrammingModelType.Cuda,
         ProgrammingModelType.Rocm,
     ),
-    Scaling(ScalingMode.Strong),
+    Scaling(ScalingMode.Strong, ScalingMode.Weak, ScalingMode.Throughput),
     Caliper,
 ):
 
@@ -31,17 +31,45 @@ class Remhos(
     variant(
         "version",
         default="develop",
-        values=("develop", "latest", "gpu-fom", "1.0"),
+        values=("develop", "latest", "1.0"),
         description="app version",
     )
 
+    variant(
+        "gpu-aware-mpi",
+        default=False,
+        values=(True, False),
+        description="Use GPU-aware MPI",
+    )
+
     maintainers("rfhaque")
 
     def compute_applications_section(self):
-        if self.spec.variants["workload"][0] == "2d":
-            self.add_experiment_variable("epm", 1024, False)
-        elif self.spec.variants["workload"][0] == "3d":
-            self.add_experiment_variable("epm", 512, False)
+        if self.spec.satisfies("exec_mode=perf"):
+            if self.spec.satisfies("workload=2d"):
+                self.add_experiment_variable("epm", 1024, True)
+                self.add_experiment_variable("o", 3, False)
+                self.add_experiment_variable("p", 14, False)
+            elif self.spec.satisfies("workload=3d"):
+                self.add_experiment_variable("epm", 512, True)
+                self.add_experiment_variable("o", 2, False)
+                self.add_experiment_variable("p", 10, False)
+        else:
+            if self.spec.satisfies("workload=2d"):
+                self.add_experiment_variable("epm", 1024, True)
+                self.add_experiment_variable("o", 3, False)
+                self.add_experiment_variable("p", 14, False)
+            elif self.spec.satisfies("workload=3d"):
+                self.add_experiment_variable("epm", 512, True)
+                self.add_experiment_variable("o", 2, False)
+                self.add_experiment_variable("p", 10, False)
+        self.add_experiment_variable("dt", -1.0, False)
+        self.add_experiment_variable("tf", 0.5, False)
+        self.add_experiment_variable("ho", 3, False)
+        self.add_experiment_variable("lo", 5, False)
+        self.add_experiment_variable("fct", 2, False)
+        self.add_experiment_variable("vs", 1, False)
+        self.add_experiment_variable("ms", 5, False)
 
         # resource_count is the number of resources used for this experiment:
         self.add_experiment_variable("resource_count", 1, False)
@@ -65,6 +93,18 @@ def compute_applications_section(self):
                     "epm": lambda var, itr, dim, scaling_factor: var.val(dim)
                     // scaling_factor,
                 },
+                ScalingMode.Weak: {
+                    "resource_count": lambda var, itr, dim, scaling_factor: var.val(dim)
+                    * scaling_factor,
+                    "epm": lambda var, itr, dim, scaling_factor: var.val(dim),
+                },
+                ScalingMode.Throughput: {
+                    "resource_count": lambda var, itr, dim, scaling_factor: var.val(
+                        dim
+                    ),
+                    "epm": lambda var, itr, dim, scaling_factor: var.val(dim)
+                    * scaling_factor,
+                },
             }
         )
 
@@ -77,8 +117,18 @@ def compute_applications_section(self):
 
         if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
             self.add_experiment_variable("n_gpus", "{n_resources}", True)
+            if self.spec.satisfies("+gpu-aware-mpi"):
+                self.add_experiment_variable("gam", "--gpu-aware-mpi")
+            else:
+                self.add_experiment_variable("gam", "--no-gpu-aware-mpi")
         else:
             self.add_experiment_variable("n_ranks", "{n_resources}", True)
 
     def compute_package_section(self):
-        self.add_package_spec(self.name, [f"remhos{self.determine_version()} +metis"])
+        gam = "~gpu-aware-mpi"
+        if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
+            if self.spec.satisfies("+gpu-aware-mpi"):
+                gam = "+gpu-aware-mpi"
+        self.add_package_spec(
+            self.name, [f"remhos{self.determine_version()} +metis {gam}"]
+        )

repos/ramble_applications/remhos/application.py

@@ -19,36 +19,57 @@ class Remhos(ExecutableApplication):
             'lagrangian','spatial-discretization','unstructured-grid',
             'network-latency-bound','network-collectives','unstructured-grid']
 
-    #TODO: add -ms flag once it's implemented
-    executable('2d', 'remhos'+' -dim 2 -epm {epm}'+' -p 14'+' -rs {rs2d}'+' -o 3 -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
-    executable('3d', 'remhos'+' -dim 3 -epm {epm}'+' -p 10'+' -rs {rs3d}'+' -o 2'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
+    executable('2d', 'remhos' +
+                     ' -dim 2 ' +
+                     ' -epm {epm}' +
+                     ' -o {o}' +
+                     ' -p {p}' +
+                     ' -dt {dt} -tf {tf}' +
+                     ' -ho {ho} -lo {lo}' +
+                     ' -fct {fct}' +
+                     ' -vs {vs}' +
+                     ' -ms {ms}' +
+                     ' -no-vis' +
+                     ' -pa' +
+                     ' -d {device}' +
+                     ' {gam}',
+                     use_mpi=True)
+
+    executable('3d', 'remhos' +
+                     ' -dim 3' +
+                     ' -epm {epm}' +
+                     ' -o {o}' +
+                     ' -p {p}' +
+                     ' -dt {dt} -tf {tf}' +
+                     ' -ho {ho} -lo {lo}' +
+                     ' -fct {fct}' +
+                     ' -vs {vs}' +
+                     ' -ms {ms}' +
+                     ' -no-vis' +
+                     ' -pa' +
+                     ' -d {device}' +
+                     ' {gam}',
+                     use_mpi=True)
+
     workload('2d', executables=['2d'])
     workload('3d', executables=['3d'])
 
     #workload_variable('mesh', default='{remhos}/data/periodic-square.mesh',
     #   description='mesh file',
     #  workloads=[''])
 
-    #workload_variable('p', default='5',
-     #   description='problem number',
-     #   workloads=['remhos'])
-
-    workload_variable('rs2d', default='4',
-        description='number of serial refinements',
-        workloads=['2d'])
-
-    workload_variable('rs3d', default='3',
-        description='number of serial refinements',
-        workloads=['3d'])
-
-    #workload_variable('rp', default='',
-     #   description='number of parallel refinements',
-      #  workloads=['remhos'])
+    workload_variable('epm', default='1024',
+        description='elements per mpi task',
+        workloads=['2d','3d'])
 
     workload_variable('o', default='2',
-        description='',
+        description='order (degree) of the finite element solution',
         workloads=['2d','3d'])
 
+    workload_variable('p', default='5',
+        description='problem number',
+        workloads=['remhos'])
+
     workload_variable('dt', default='-1.0',
         description='time step',
         workloads=['2d','3d'])
@@ -76,8 +97,13 @@ class Remhos(ExecutableApplication):
     workload_variable('ms', default='5',
         description='ms',
         workloads=['2d','3d'])
+
     workload_variable('device', default='cpu',
-        description='cpu or cuda',
+        description='cpu, cuda or hip',
+        workloads=['2d','3d'])
+
+    workload_variable('gam', default='--no-gpu-aware-mpi',
+        description='--gpu-aware-mpi or --no-gpu-aware-mpi',
         workloads=['2d','3d'])
 
     figure_of_merit("FOM", log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'FOM:\s+(?P<fom>[0-9]*\.[0-9]*)', group_name='fom', units='megadofs x time steps / second')

repos/spack_repo/benchpark/packages/remhos/package.py

@@ -19,38 +19,43 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage):
     tags = ["proxy-app"]
 
     homepage = "https://github.com/CEED/Remhos"
-    url = "https://github.com/CEED/Remhos/archive/v1.0.tar.gz"
     git = "https://github.com/CEED/Remhos.git"
 
     maintainers("v-dobrev", "tzanio", "vladotomov")
 
     license("BSD-2-Clause")
 
     version("develop", branch="master")
-    version("gpu-fom", branch="gpu-fom")
     version("1.0", sha256="e60464a867fe5b1fd694fbb37bb51773723427f071c0ae26852a2804c08bbb32")
 
     variant("metis", default=True, description="Enable/disable METIS support")
     variant("caliper", default=False, description="Enable/disable Caliper support")
+    variant("gpu-aware-mpi", default=False, description="Enable GPU aware MPI")
+    variant("raja", default=True, description="Use RAJA backend for MFEM")
 
     depends_on("c", type="build")
     depends_on("cxx", type="build")
     depends_on("fortran", type="build")
 
     depends_on("mfem+mpi+metis", when="+metis")
     depends_on("mfem+mpi~metis", when="~metis")
+    depends_on("mfem+raja", when="+raja")
     depends_on("caliper", when="+caliper")
     depends_on("adiak~shared", when="+caliper")
 
     depends_on("zlib+optimize+pic~shared")
     depends_on("mfem@develop", when="@develop")
     depends_on("mfem@4.1.0:", when="@1.0")
-    depends_on("mfem@develop", when="@gpu-fom")
-    depends_on("mfem@develop", when="@gpu-opt")
     depends_on("mfem+caliper", when="+caliper")
     depends_on("mfem cxxstd=17")
 
     requires("^[virtuals=zlib-api] zlib")
+
+    depends_on("mpi")
+    depends_on("hypre+mpi")
+    depends_on("hypre+mixedint~fortran")
+    depends_on("hypre+caliper", when="+caliper")
+
     depends_on("hypre+cuda+mpi+umpire", when="+cuda")
     depends_on("hypre~cuda", when="~cuda")
     depends_on("mfem+cuda+mpi+umpire", when="+cuda")
@@ -71,7 +76,11 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage):
         depends_on("mfem amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch))
         depends_on("umpire amdgpu_target={0}".format(arch), when="amdgpu_target={0}".format(arch))
 
-    depends_on("hypre+gpu-aware-mpi", when="^cray-mpich+gtl")
+    depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi")
+
+    def setup_run_environment(self, env):
+        if "+gpu-aware-mpi" in self.spec:
+            env.set("MFEM_GPU_AWARE_MPI", "1")
 
     def setup_build_environment(self, env):
         if "+cuda" in self.spec: