#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015-2023, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8   ****************************************

#[ICS VERSION STRING: unknown]

# PURPOSE:
#
# This file sets environment variables for an openmpi job.
# Note that there are many, many such variables.
# See http://www.open-mpi.org/faq/?category=openfabrics for details, or use
# the command:
#
# $ ompi_info
#
# to get a list of all current settings.

# SYNTAX:
#
# This file must be a valid BASH script. In general, anything that's valid
# in BASH is valid here.
#
# To pass OMPI_*, FI_*, PSM3_* variables to openmpi, they must be specified as:
#	export VARIABLE_NAME=$(VARIABLE_NAME:-variablevalue}
# and our caller will add these to MPI_CMD_ARGS.
#
# any other variables must be specified as:
#	export MPI_CMD_ARGS="$MPI_CMD_ARGS -x VARIABLE_NAME ${VARIABLE_NAME:-variablevalue}"
#
# any other settings must be specified as:
#	export MPI_CMD_ARGS="$MPI_CMD_ARGS <settings>"
#
# The precedence if a setting is specified in multiple ways is as follows:
# 1. settings added to MPI_CMD_ARGS in this file
#    these will be placed at the end of the mpirun command line.
# 2. variables added to MPI_CMD_ARGS in this file with an
#    explicit value such as -x variable=value
#    without using the ${variable:-value} syntax
# 3. explicit settings of variables in this file
#    such as variable=value
#    without using the ${variable:-value} syntax
# 4. variables set in the psm3.params file with an
#    explicit value such as variable=value
#    without using the ${variable:-value} syntax
# 5. env variables passed into the job such as an export immediately
#    before starting the job or on the job command line.
#    Such as: variable=value ./run_bw5
# 6. env variables set by the system or user.  Such as via .bashrc or
#    explicitly loaded "modules files"
# 7. variables set in the psm3.params file with the ${variable:-value} syntax
# 8. variables set in this file with the ${variable:-value} syntax
# 9. settings in /etc/psm3.conf
# If a variable is specified multiple ways, lowest number above wins.
#
# The logfile created by the job will show relevant env variables as well
# as the mpirun command line.
#
# The ${varname:-value} syntax allows this file to set values typically used
# while still permitting the caller or user to export a different value
# when it's desired to override the value specified in this file.

# SAMPLE Tuning variables:
#
# Uncomment the following lines to enable them.
#
export MPI_CMD_ARGS="$MPI_CMD_ARGS -mca plm_rsh_no_tree_spawn 1"

# This disables the OFI BTL in order to prevent duplicate OFI contexts from being allocated when
# they are not needed
export MPI_CMD_ARGS="$MPI_CMD_ARGS --mca btl ^ofi"

# This defines where external providers are loaded from
export FI_PROVIDER_PATH=${FI_PROVIDER_PATH:-/usr/lib64/libfabric:/usr/lib/x86_64-linux-gnu/libfabric}

#export FI_PROVIDER=${FI_PROVIDER:-psm3}
# exclude all but the psm3 provider this includes excluding the psm3;ofi_rxd layered provider as it is not performant
export FI_PROVIDER="${FI_PROVIDER:-^$(fi_info | sed -n 's/^provider: //p' | sort -u | grep -v 'psm3$' | tr '\n' ',' | sed 's/,$//')}"

# It is recommended to use the OFI interface over PSM3 within OpenMPI
export MPI_CMD_ARGS="$MPI_CMD_ARGS -mca mtl ofi"
