1. Login to Frontier:

   ssh frontier.olcf.ornl.gov

2. Modules to load:

   module load amd-mixed

3. CMake Build/Install

   cmake -Bbuild_hip -DNWPW_HIP=ON ./Nwpw -DGPU_TARGETS=gfx90a
   cd build_hip
   make -j

4. Job submission script via sbatch job_submit.sbatch ```

   !/bin/bash

SBATCH -A

SBATCH -J

SBATCH -o %x-%j.out

SBATCH -t 00:15:00

SBATCH -N 1

SBATCH -C nvme

SBATCH –mail-user=

SBATCH –mail-type=END

module load amd-mixed module list

export MPICH_GPU_SUPPORT_ENABLED=0 export OMP_NUM_THREADS=1 export CRAYPE_LINK_TYPE=dynamic

date

NNODES=1 NRANKS_PER_NODE=8 NTOTRANKS=$(( NNODES * NRANKS_PER_NODE ))

PWDFT_EXE= PWDFT_INPUT=

srun -N$NNODES-n\{NNODES\}\; -n${NTOTRANKS} -c1 –ntasks-per-gpu=1 –gpus-per-node=8 –gpu-bind=closest $PWDF{T}_{E}XE\{PWDFT\_EXE\}${PWDFT_INPUT}

</details>

##  Instructions for NERSC Perlmutter
<details>
<summary>Toggle for Details</summary>

CMake Build/Install

cmake -H. -Bbuild_cuda -DNWPW_CUDA=ON ./Nwpw -DCMAKE_CUDA_ARCHITECTURES=80 -DCUDA_cublas_LIBRARY=$CRA{Y}_{C}UDATOOLKI{T}_{D}IR\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}mat{h}_{l}ibs\mathrm{/}\{CRAY\_CUDATOOLKIT\_DIR\}/../../math\_libs/${CRAY_CUDATOOLKIT_VERSION#_}/lib64/libcublas.so -DCUDA_cufft_LIBRARY=$CRA{Y}_{C}UDATOOLKI{T}_{D}IR\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}mat{h}_{l}ibs\mathrm{/}\{CRAY\_CUDATOOLKIT\_DIR\}/../../math\_libs/${CRAY_CUDATOOLKIT_VERSION#}/lib64/libcufft.so -DCUDA_cusolver_LIBRARY=$CRA{Y}_{C}UDATOOLKI{T}_{D}IR\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}\mathrm{.}\mathrm{.}\mathrm{/}mat{h}_{l}ibs\mathrm{/}\{CRAY\_CUDATOOLKIT\_DIR\}/../../math\_libs/${CRAY_CUDATOOLKIT_VERSION#*}/lib64/libcusolver.so cd build_cuda make

</details>

##  Build Instructions on NERSC Cori-Haswell

### required modules on Cori-Haswell

module purge module load cmake/3.20.2 module load PrgEnv-intel module load craype-haswell module load openmpi

module purge;
module load cmake/3.20.2;
module load PrgEnv-intel;
module load craype-haswell;
module load openmpi

### Build Instructions on Cori-Haswell (starting from PWDFT directory)

mkdir build cd build cmake -DCMAKE_CXX_COMPILER=CC ../Nwpw/

### Running on Cori-Haswell

salloc –nodes 1 –qos interactive –time 01:00:00 –constraint haswell srun -n -c a.out srun -n -c pwdft

### runnin on haswell??

salloc –nodes 1 –qos interactive –time 01:00:00 –constraint haswell cd PWDFT/QA/C2_steepest_descent srun -n 24 ../../build/pwdft c2-sd.nw

##  Instructions for ALCF Polaris
<details>
<summary>Toggle for Details</summary>

1. Login to Frontier:

ssh polaris.alcf.anl.gov

2. Modules to load:

module load PrgEnv-gnu cudatoolkit-standalone cmake

3. CMake Build/Install

cmake -Bbuild_cuda -DNWPW_CUDA=ON ./Nwpw -DCMAKE_CUDA_ARCHITECTURES=80 cd build_cuda make -j

4. Job submission script via `qsub polaris_submit.pbs`. Please note that it needs an additional bash-script to bind the GPU-IDs to MPI-ranks.
The script below is the MPI-ranks with GPUs affinity script:

!/bin/bash

num_gpus=4

need to assign GPUs in reverse order due to topology

See Polaris Device Affinity Information https://www.alcf.anl.gov/support/user-guides/polaris/hardware-overview/machine-overview/index.html

gpu=$((((${num_gpus} - 1 - $PM{I}_{L}OCA{L}_{R}ANK\{PMI\_LOCAL\_RANK\}\; \%${num_gpus}))

unset CUDA_VISIBLE_DEVICES if [ $PM{I}_{L}OCA{L}_{R}ANK-ne4];thenexportCUD{A}_{V}ISIBL{E}_{D}EVICES=\{PMI\_LOCAL\_RANK\}\; -ne\; 4\; ];\; then\; export\; CUDA\_VISIBLE\_DEVICES=$gpu fi #echo “RANK= $PM{I}_{R}ANKLOCA{L}_{R}ANK=\{PMI\_RANK\}\; LOCAL\_RANK=${PMI_LOCAL_RANK} gpu= $CUD{A}_{V}ISIBL{E}_{D}EVICES\mathrm{"}exec\mathrm{"}\{CUDA\_VISIBLE\_DEVICES\}"\; exec\; "$@”

The actual job-script is below:

!/bin/bash

PBS -N develop

PBS -l select=8:system=polaris

PBS -l place=scatter

PBS -l walltime=01:00:00

PBS -l filesystems=home:eagle

PBS -A

PBS -q workq

module load PrgEnv-gnu cudatoolkit-standalone module list

export MPICH_GPU_SUPPORT_ENABLED=0 export CRAYPE_LINK_TYPE=dynamic env nvidia-smi topo -m

cd ${PBS_O_WORKDIR}

NNODES=wc -l < $PBS_NODEFILE NRANKS_PER_NODE=4 NTHREADS=1

NTOTRANKS=$((NNODES\ast NRANK{S}_{P}E{R}_{N}ODE))echo\mathrm{"}NU{M}_O{F}_{N}ODES=((\; NNODES\; *\; NRANKS\_PER\_NODE\; ))\; echo\; "NUM\_OF\_NODES=${NNODES} TOTAL_NUM_RANKS= $NTOTRANKSRANK{S}_{P}E{R}_{N}ODE=\{NTOTRANKS\}\; RANKS\_PER\_NODE=${NRANKS_PER_NODE} THREADS_PER_RANK= ${NTHREADS}”

PWDFT_EXE= PWDFT_INPUT=

mpiexec -n $NTOTRANKS--ppn\{NTOTRANKS\}\; --ppn${NRANKS_PER_NODE} –mem-bind list:0:1:2:3 –cpu-bind list:0-7:8-15:16-23:24-31 –env OMP_NUM_THREADS=$NTHREADS\mathrm{.}\mathrm{/}gp{u}_{b}in{d}_{a}ffinity\mathrm{.}sh\{NTHREADS\}\; ./gpu\_bind\_affinity.sh${PWDFT_EXE} ${PWDFT_INPUT}

</details>

## Build instructions on NERSC Cori-CUDA

### Required Modules on Cori-CUDA

module unload impi module load PrgEnv-intel module load cmake module load cudatoolkit

### Build Instructions on Cori-CUDA (starting from PWDFT directory)

mkdir build_cuda cd build-cuda cmake -DNWPW_CUDA=ON ../Nwpw/

### Running on Cori-CUDA

module load cgpu salloc -C gpu -t 60 -c 10 -G 1 -q interactive -A salloc -C gpu -t 60 -c 10 -G 1 -q interactive -A mp119

# Making shared library
To generate a library clean the build directory and then regenerate cmake with

cmake ../Nwpw -DMAKE_LIBRARY=true

on Linux use:

cmake ../Nwpw -DMAKE_LIBRARY=true -DCMAKE_POSITION_INDEPENDENT_CODE=ON

Compile and generate the shared library in the build directory.

make

The shared library, libpwdft.dylib, should be generated and present in the build directory.

prompt% ls CMakeCache.txt Makefile cmake_install.cmake nwpwlib/ CMakeFiles/ NwpwConfig.h libpwdft.dylib* pspw/

Example header to make function calls

include

include “mpi.h”

namespace pwdft { using namespace pwdft;

extern char *util_date(); extern void seconds(double *); extern int cpsd(MPI_Comm, std::string&); extern int cpmd(MPI_Comm, std::string&); extern int pspw_minimizer(MPI_Comm, std::string&); extern int pspw_geovib(MPI_Comm, std::string&); }

Example function call

ierr += pwdft::pspw_geovib(MPI_COMM_WORLD,nwinput);

## Using shared library on a MAC

mpic++ test.cpp ../build-shared/libpwdft.dylib setenv DYLD_LIBRARY_PATH /Users/bylaska/Codes/PWDFT/build-shared a.out

## Using shared library on LINUX

mpic++ test.cpp ../build-shared/libpwdft.so setenv DYLD_LIBRARY_PATH /Users/bylaska/Codes/PWDFT/build-shared a.out

 [2:17 PM] Bagusetty, Abhishek
 add_library(pwdft SHARED nwpw.cpp)

 [2:18 PM] Bagusetty, Abhishek
 CMakeLists.txt (right after this line add_executable(pwdft nwpw.cpp))

---

<p align="center">
  <img src="https://raw.githubusercontent.com/ebylaska/PWDFT/gh-pages/assets/PWDFT-LOGO.png"
       alt="Hematite–water interface"
       style="width:30%; min-width:250px; max-width:700px; border-radius:40%; box-shadow:0 0 10px rgba(0,0,0,0.3); margin:10px;">
</p>

<p align="center"><em>Figure: Hematite–water interface showing layered hydration and hydroxylation structure.</em></p>