Script for building octopus on MPCDF systems

The following script can be used to compile octopus on MPCDF systems. The latest version of this compilation script is available in the octopus git repository at scripts/build/build_octopus_mpcdf.sh path. By default, this script expects to be run in a subdirectory (e.g. _build) of the root octopus directory. The script compiles the octopus for CPU (with MPI and OpenMP support) using GNU compilers, linking it to OpenMPI. You can compile the code for GPU using --cuda flag. The compiler, and the MPI library can also be changed using the CLI. At the end of compilation a list of “required runtime modules” will be printed on screen. These modules must be loaded in your job submisstion script before running the compiled octopus binary for your calculations. A copy of this compilation script, its CLI arguments, config.log, and the list of required runtime modules will be all stored at .build.doc/ in installation path for your future reference. To see a complete list of available options and their default values, run the script with --help flag.

#!/bin/bash
#
# Compilation script for octopus. If this script is not executed in a subdirectory (e.g. _build) of the root octopus directory,
# you have to pass the path to octopus directory to the compilation script using the --root-path flag
# See ./build_octopus.sh --help for complete list of available options.
#
# Usage examples:
#
# build octopus for CPU, using gnu compilers and openmpi, with octopus at "$PWD/.." and install it to "$PWD/installed"
# > ./build_octopus_mpcdf.sh
#
# build octopus for CPU, using intel compilers and intel-mpi, install it to "/u/user/octopus/"
# > ./build_octopus_mpcdf.sh --compiler=intel --mpi=impi --prefix=/u/user/octopus/
#
# build octopus with no compiler optimization, enable compiler warnings, and runtime checks
# > ./build_octopus_mpcdf.sh --debug
#
# build octopus for CPU without MPI (with OpenMP support only)
# > ./build_octopus_mpcdf.sh --no-mpi
#
# build octopus for CPU + GPU, using gnu compilers and cuda-aware openmpi (if possible)
# > ./build_octopus_mpcdf.sh --cuda
#
# build octopus for CPU + GPU, using gnu compilers and cuda-unaware(!) openmpi
# > ./build_octopus_mpcdf.sh --cuda --no-cuda-aware-mpi
#
# rebuild without running the configuration script
# > ./build_octopus_mpcdf.sh --no-config
#
# rebuild without compiling the dependencies (WARNING: DO NOT USE THIS AFTER CHANGING THE FUNCTION/SUBROUTINE INTERFACES)
# > ./build_octopus_mpcdf.sh --no-dep
#
#
# NOTE: You must load the required runtime modules in your job submission script before
#       executing the octopus for your calculations. The list of these modules is printed
#       at the end of compilation and also is saved in your installation path at
#       .build.doc/required_runtime_modules file. You can also generate this list by passing
#       '--show-modules-only' flag to this script without compiling the code.

set -e

# version of this script
VERSION="1.5"


# default value of cli arguments
_arg_config="on"
_arg_dep="on"
_arg_cuda="off"
_arg_cuda_aware_mpi="on"
_arg_debug="off"
_arg_compiler="gnu"
_arg_mpi="openmpi"
_arg_prefix="$PWD/installed"
_arg_root_path="${PWD%/*}"
_arg_show_modules_only="off"


# default modules
# Latest tests:
## by MFT on COBRA, RAVEN & ADA clusters @01.2023 using octopus: fd2983011b37c086e7fd979e99afef197beab7a7
# other clusters may have different version of these modules
# update the versions if necessary!
#
## for OpenMP only
modules_gcc="gcc/11"
modules_intel="intel/21.5.0"
modules_libs_openmp="mkl/2022.0 gsl hdf5-serial netcdf-serial etsf_io libxc/5.1.5 libvdwxc cgal boost/1.74"
## for MPI
modules_intel_impi="intel/21.5.0 impi/2021.5"
modules_gcc_impi="gcc/11 impi/2021.5"
modules_gcc_openmpi="gcc/11 openmpi/4"
modules_libs_mpi="mkl/2022.0 gsl hdf5-serial netcdf-serial etsf_io libxc/5.1.5 libvdwxc-mpi cgal boost/1.74 elpa/mpi/standard/2022.11.001 metis-64 parmetis-64 dftbplus/22.1"
## For GPU
modules_cuda="cuda/11.4"
modules_cuda_aware_mpi="openmpi_gpu/4"


# configuration features to verify
## common features we expect to detect in all cases
common_features_list=('CGAL' 'NETCDF' 'ETSF_IO' 'FFTW3' 'LIBXC5' 'LIBXC_FXC' 'LIBXC_KXC' 'LIBVDWXC' 'OPENMP')
## expected features of mpi version
mpi_features_list=('MPI' 'ELPA' 'PARMETIS' 'LIBVDWXC_MPI' 'DFTBPLUS')
## expected features of cuda version
cuda_features_list=('CUDA')

expected_features_list=( "${common_features_list[@]}" )


die()
{
  _die_ret="${2:-1}"
  test "${_PRINT_HELP:-no}" = yes && print_help >&2
  echo "$1" >&2
  exit "${_die_ret}"
}

print_help()
{
  printf '%s\n' "Compilation script for octopus. This script needs to be executed in a subdirectory (e.g. _build) of the root directory or the path to octopus root directory must be set using --root-path flag."
  print_usage
}

print_usage()
{
  printf '\nUsage: %s [--(no-)config] [--(no-)dep] [--(no-)cuda] [--(no-)cuda-aware-mpi] [--(no-)debug] [--compiler <arg>] [--no-mpi] [--mpi <arg>] [--prefix <arg>] [--show-modules-only] [-v|--version] [-h|--help]\n' "$0"
  printf '\t%s\n' "--config, --no-config: run configure script (on by default)"
  printf '\t%s\n' "--dep, --no-dep: compile dependencies (on by default)"
  printf '\t%s\n' "--cuda, --no-cuda: compile with CUDA support (off by default)"
  printf '\t%s\n' "--cuda-aware-mpi, --no-cuda-aware-mpi: compile with CUDA aware MPI support (on by default)"
  printf '\t%s\n' "--debug, --no-debug: compile with debug flags [disable optimization, activate compiler warnings and runtime checks, link to debug impi] (off by default)"
  printf '\t%s\n' "--compiler: compiler type. available options: 'intel' or 'gnu' (default: 'gnu')"
  printf '\t%s\n' "--no-mpi: compile without the MPI support (equivalent to '--mpi=off')"
  printf '\t%s\n' "--mpi: MPI implementation. available options: 'impi', 'openmpi', or 'off' (default: 'openmpi')"
  printf '\t%s\n' "--prefix: installation path (default: './installed/')"
  printf '\t%s\n' "--root-path: path to the root of octopus repository (default: '../')"
  printf '\t%s\n' "--show-modules-only: print the required runtime modules and exit"
  printf '\t%s\n' "-v, --version: print compilation script version"
  printf '\t%s\n' "-h, --help: print help (this list!)"
}


parse_commandline()
{
  while test $# -gt 0
  do
    _key="$1"
    case "$_key" in
      --no-config|--config)
        test "${1:0:5}" = "--no-" && _arg_config="off"
        ;;
      --no-dep|--dep)
        test "${1:0:5}" = "--no-" && _arg_dep="off"
        ;;
      --no-cuda|--cuda)
        test "${1:0:5}" = "--no-" || _arg_cuda="on"
        ;;
      --no-cuda-aware-mpi|--cuda-aware-mpi)
        test "${1:0:5}" = "--no-" && _arg_cuda_aware_mpi="off"
        ;;
      --no-debug|--debug)
        test "${1:0:5}" = "--no-" || _arg_debug="on"
        ;;
      --compiler)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_compiler="$2"
        shift
        ;;
      --compiler=*)
        _arg_compiler="${_key##--compiler=}"
        ;;
      --no-mpi)
        _arg_mpi="off"
        ;;
      --mpi)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_mpi="$2"
        shift
        ;;
      --mpi=*)
        _arg_mpi="${_key##--mpi=}"
        ;;
      --prefix)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_prefix="$2"
        shift
        ;;
      --prefix=*)
        _arg_prefix="${_key##--prefix=}"
        ;;
      --root-path)
        test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1
        _arg_root_path="$2"
        shift
        ;;
      --root-path=*)
        _arg_root_path="${_key##--root-path=}"
        ;;
      --show-modules-only)
        _arg_show_modules_only="on"
        ;;
      -v|--version)
        echo Octopus compilation script v$VERSION
        exit 0
        ;;
      -h|--help)
        print_help
        exit 0
        ;;
      -h*)
        print_help
        exit 0
        ;;
      *)
        _PRINT_HELP=yes die "FATAL ERROR: Unexpected argument '$1'" 1
        ;;
    esac
    shift
  done
}

# verify expected features are detected in config.h as "#define HAVE_*** 1"
verify_configuration()
{
  read -ra _expected_features <<< "$@"

  for feature in "${_expected_features[@]}"; do
    printf "checking feature: %s ..." "$feature"
    _definition="#define HAVE_$feature 1"
    grep -q "$_definition" config.h || ( echo "NOT FOUND!" && echo "ERROR: Configuration script failed to detect $feature" && exit 1 )
    echo "OK!"
  done

}

parse_commandline "$@"

# expand ~ in path arguments
_arg_prefix="${_arg_prefix/#\~/$HOME}"
_arg_root_path="${_arg_root_path/#\~/$HOME}"

echo "====================================="
[ "$_arg_show_modules_only" = "off" ] && echo "run configure: $_arg_config"
[ "$_arg_show_modules_only" = "off" ] && echo "build dependencies: $_arg_dep"
echo "compiler: $_arg_compiler"
echo "mpi: $_arg_mpi"
echo "cuda support: $_arg_cuda"
[ "$_arg_mpi" = "openmpi" ] && [ "$_arg_cuda" = "on" ] && echo "cuda-aware-mpi: $_arg_cuda_aware_mpi"
[ "$_arg_show_modules_only" = "off" ] && echo "debug mode: $_arg_debug"
[ "$_arg_show_modules_only" = "off" ] && echo "installation path: $_arg_prefix"
[ "$_arg_show_modules_only" = "off" ] && echo "octopus repository path: $_arg_root_path"
echo "====================================="

if [ ! -f "$_arg_root_path/configure.ac" ]; then
  echo "Error: Could not find configure.ac in octopus repository path ($_arg_root_path). Please pass the right path to the compilation script using the \"--root-path\" flag"
  exit 1
fi


# validating script arguments

if [ "$_arg_compiler" != "intel" ] && [ "$_arg_compiler" != "gnu" ]; then
  echo "Error: Invalid --compiler option: $_arg_compiler"
  print_usage
  exit 1
fi

if [ "$_arg_mpi" != "impi" ] && [ "$_arg_mpi" != "openmpi" ] && [ "$_arg_mpi" != "off" ]; then
  echo "Error: Invalid --mpi option: $_arg_mpi"
  print_usage
  exit 1
fi

if [ "$_arg_compiler" = "intel" ] && [ "$_arg_mpi" = "openmpi" ]; then
  echo "WARNING: Compiling Octopus with intel compilers and linking to OpenMPI library is not supported!"
  echo "Will use IntelMPI (impi) instead!"
  _arg_mpi="impi"
fi



module purge

if [ "$_arg_compiler" = "intel" ]; then
  if [ "$_arg_mpi" = "off" ]; then
    # shellcheck disable=2086
    module load $modules_intel
    required_modules+=" ${modules_intel}"

    export CC=icc
    export FC=ifort
    export CXX=icpc
    MKL_FLAGS="-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -liomp5 -lpthread -lm -ldl"

  elif [ "$_arg_mpi" = "impi" ]; then
    # shellcheck disable=2086
    module load $modules_intel_impi
    required_modules+=" ${modules_intel_impi}"

    if [ "$_arg_debug" = "on" ]; then
      export I_MPI_LINK=dbg
    else
      export I_MPI_LINK=opt
    fi

    export CC=mpiicc
    export FC=mpiifort
    export CXX=mpiicpc
    MKL_FLAGS="-lmkl_scalapack_lp64 -lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -liomp5 -lpthread -lm -ldl"

  elif [ "$_arg_mpi" = "openmpi" ]; then
    # we should never reach this part of script! Adapt the script if we support Intel compilers + OpenMPI in future.
    echo "ERROR: Compiling Octopus with intel compilers and linking to OpenMPI library is not currently supported!"
    exit 1
  fi

  if [ "$_arg_debug" = "on" ]; then
    export CFLAGS="-O0 -g -traceback"
    FCFLAGS_EXTRA="-warn all -warn noexternals -warn nounused -check all"
  else
    export CFLAGS="-O3 -xCORE-AVX512 -qopt-zmm-usage=high -fma -g -traceback"
  fi
  export CXXFLAGS="$CFLAGS"
  export FCFLAGS="$CFLAGS $FCFLAGS_EXTRA"

elif [ "$_arg_compiler" = "gnu" ]; then
  if [ "$_arg_mpi" = "off" ]; then

    module load $modules_gcc
    required_modules+=" ${modules_gcc}"

    export CC=gcc
    export FC=gfortran
    export CXX=g++
    MKL_FLAGS="-lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread -lm -ldl"

  elif [ "$_arg_mpi" = "impi" ]; then
    # shellcheck disable=2086
    module load $modules_gcc_impi
    required_modules+=" ${modules_gcc_impi}"

    export CC=mpigcc
    export FC=mpigfortran
    export CXX=mpig++
    MKL_FLAGS="-lmkl_scalapack_lp64 -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_intelmpi_lp64 -lgomp -lpthread -lm -ldl"

  elif [ "$_arg_mpi" = "openmpi" ]; then
    # shellcheck disable=2086
    module load $modules_gcc_openmpi
    required_modules+=" ${modules_gcc_openmpi}"

    export CC=mpicc
    export FC=mpif90
    export CXX=mpicxx
    MKL_FLAGS="-lmkl_scalapack_lp64 -lmkl_gf_lp64 -lmkl_gnu_thread -lmkl_core -lmkl_blacs_openmpi_lp64 -lgomp -lpthread -lm -ldl"
  fi

  if [ $_arg_debug = on ]; then
    export CFLAGS="-O0 -g"
    FCFLAGS_EXTRA="-fallow-argument-mismatch -fallow-invalid-boz -fcheck=all -Wall -Wno-c-binding-type -Wno-maybe-uninitialized -Wno-unused-dummy-argument"
  else
    export CFLAGS="-O3 -march=native -g"
    FCFLAGS_EXTRA="-fallow-argument-mismatch -fallow-invalid-boz"
  fi
  export CXXFLAGS="$CFLAGS"
  export FCFLAGS="$CFLAGS $FCFLAGS_EXTRA"
fi

if [ "$_arg_mpi" = "off" ]; then
  # shellcheck disable=2086
  module load $modules_libs_openmp
  export MKL="-L${MKLROOT}/lib/intel64 ${MKL_FLAGS}"
  ENABLE_MPI=
  SCALAPACK=no
  PARMETIS_HOME=no
  METIS_HOME=no
  ELPA_HOME=no
  DFTBPLUS_HOME=no
else
  # shellcheck disable=2086
  module load $modules_libs_mpi
  export MKL="-L${MKLROOT}/lib/intel64 ${MKL_FLAGS}"
  expected_features_list+=( "${mpi_features_list[@]}" )
  ENABLE_MPI=--enable-mpi
  SCALAPACK=$MKL

fi


if [ "$_arg_compiler" = "intel" ]; then
  # gcc 10 or later is needed for newer C++ libraries. This needs to be loaded after $modules_libs_...
  module load gcc/11
fi

if [ "$_arg_cuda" = "on" ]; then
  module load $modules_cuda
  required_modules+=" ${modules_cuda}"
  expected_features_list+=( "${cuda_features_list[@]}" )

  if [ "$_arg_mpi" = "openmpi" ] && [ "$_arg_cuda_aware_mpi" = "on" ] ; then
    # on COBRA the underlying network drivers do not support CUDA-aware MPI
    module -s load $modules_cuda_aware_mpi && required_modules+=" ${modules_cuda_aware_mpi}"\
    || echo "WARNING: CUDA-aware MPI module does not seem to be available in the current environment. Proceeding without the CUDA-aware MPI support!"
  fi
  CUDA_FLAGS="--enable-cuda --enable-nvtx --with-cuda-prefix=$CUDA_HOME"
  LDFLAGS="$LDFLAGS:$CUDA_HOME/lib64"
else
  CUDA_FLAGS=""
fi

if [ "$_arg_show_modules_only" = "on" ]; then
  echo -e "\nRequired runtime modules for the selected configuration would be: ${required_modules}\n"
  exit 0
fi

module list

export CXXFLAGS="$CXXFLAGS -std=c++14"
export LDFLAGS="-Xlinker -rpath=$MKL_HOME/lib/intel64:$GSL_HOME/lib:$NETCDF_HOME/lib:$ELPA_HOME/lib:$METIS_HOME/lib:$PARMETIS_HOME/lib:$LIBXC_HOME/lib:$DFTBPLUS_HOME/lib"

if [ "$_arg_config" != "off" ]; then
  pushd "$_arg_root_path"
  autoreconf -i
  popd

  # shellcheck disable=2086
  "$_arg_root_path"/configure $CUDA_FLAGS \
  FCFLAGS_FFTW="-I$MKLROOT/include/fftw" \
  --prefix="$_arg_prefix" \
  $ENABLE_MPI \
  --enable-openmp \
  --disable-gdlib \
  --enable-shared --disable-static \
  --with-gsl-prefix="$GSL_HOME" \
  --with-cgal="$CGAL_HOME" \
  --with-libxc-prefix="$LIBXC_HOME" \
  --with-libvdwxc-prefix="$LIBVDWXC_HOME" \
  --with-blas="$MKL" \
  --with-lapack="$MKL" \
  --with-blacs="$SCALAPACK" \
  --with-scalapack="$SCALAPACK" \
  --with-netcdf-prefix="$NETCDF_HOME" \
  --with-etsf-io-prefix="$ETSF_IO_HOME" \
  --with-metis-prefix="$METIS_HOME" \
  --with-parmetis-prefix="$PARMETIS_HOME" \
  --with-boost="$BOOST_HOME" \
  --with-elpa-prefix="$ELPA_HOME" \
  --with-dftbplus-prefix="$DFTBPLUS_HOME"

  verify_configuration "${expected_features_list[@]}"
fi


if [ "$_arg_dep" = "off" ]; then
  NODEP=1
else
  NODEP=0
fi

make NODEP=$NODEP -j20
make NODEP=$NODEP -j20 install

# keep the compilation script, its cli arguments, config.log, and the list of required runtime modules
# at .build.doc/ in installation path for future reference
mkdir -p "$_arg_prefix"/.build.doc/
cp -f config.log "$_arg_prefix"/.build.doc/
cp -f "$0" "$_arg_prefix"/.build.doc/
echo "$0 $*" > "$_arg_prefix"/.build.doc/compilation_command
echo "$required_modules" > "$_arg_prefix"/.build.doc/required_runtime_modules
echo "====================================="
echo "Compilation done!"
echo "A copy of this compilation script, its CLI arguments, config.log, and the list of required runtime modules are all stored in $_arg_prefix/.build.doc/ for future reference"
echo -e "\nRequired runtime modules: ${required_modules}\n"