#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015-2023, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8   ****************************************

#[ICS VERSION STRING: unknown]

# PURPOSE:
#
# This file sets environment variables for an IntelMPI job.
# Note that there are many, many such variables.
#
# SYNTAX:
#
# This file must be a valid BASH script. In general, anything that's valid
# in BASH is valid here.

# To pass variables to IntelMPI, they should be specified as:
#	export VARIABLE_NAME=$(VARIABLE_NAME:-variablevalue}
#
# any other settings must be specified as:
#	export MPI_CMD_ARGS="$MPI_CMD_ARGS <settings>"
#
# The precedence if a setting is specified in multiple ways is as follows:
# 1. settings added to MPI_CMD_ARGS in this file
#    these will be placed at the end of the mpirun command line.
# 2. variables added to MPI_CMD_ARGS in this file with an
#    explicit value such as -genv variable=value
#    without using the ${variable:-value} syntax
# 3. explicit settings of variables in this file
#    such as variable=value
#    without using the ${variable:-value} syntax
# 4. variables set in the psm3.params file with an
#    explicit value such as variable=value
#    without using the ${variable:-value} syntax
# 5. env variables passed into the job such as an export immediately
#    before starting the job or on the job command line.
#    Such as: variable=value ./run_bw5
# 6. env variables set by the system or user.  Such as via .bashrc or
#    explicitly loaded "modules files"
# 7. variables set in the psm3.params file with the ${variable:-value} syntax
# 8. variables set in this file with the ${variable:-value} syntax
# 9. settings in /etc/psm3.conf
# If a variable is specified multiple ways, lowest number above wins.
#
# The logfile created by the job will show relevant env variables as well
# as the mpirun command line.
#
# The ${varname:-value} syntax allows this file to set values typically used
# while still permitting the caller or user to export a different value
# when it's desired to override the value specified in this file.

# SAMPLE Tuning variables:
#
# Uncomment the following lines to enable them.
#

# It is recommended to use the OFI interface over PSM3 within IntelMPI
# These variables are passed to the Intel mpirun command.
export I_MPI_FABRICS=${I_MPI_FABRICS:-'shm:ofi'}

# For Intel MPI v2021.6.0 it's recommended to use FI_PROVIDER
# for other releases I_MPI_OFI_PROVIDER is recommended
export I_MPI_OFI_PROVIDER=${I_MPI_OFI_PROVIDER:-psm3}
#export FI_PROVIDER=${FI_PROVIDER:-psm3}

# This defines where external providers are loaded from
# Typically must explicitly set as Intel MPI vars.sh will export it before this
#export FI_PROVIDER_PATH=/usr/lib64/libfabric:/usr/lib/x86_64-linux-gnu/libfabric
#export FI_PROVIDER_PATH=${FI_PROVIDER_PATH:-/usr/lib64/libfabric:/usr/lib/x86_64-linux-gnu/libfabric}

# When running an Intel GPU application these are required
#export EnableImplicitScaling=${EnableImplictScaling:-0} NEOReadDebugKeys=${NEOReadDebugKeys:-1}

# When running a GPU application these settings are recommended to take full
# advantage of Direct GPU Access and RDMA.  Typically I_MPI_OFFLOAD_RDMA 
# (which enables PSM3_GPUDIRECT) offers better performance, but also depends
# on the rv kernel module for GPU pinning
#export I_MPI_OFFLOAD=${I_MPI_OFFLOAD:-1} I_MPI_OFFLOAD_PIPELINE=${I_MPI_OFFLOAD_PIPELINE:-0} I_MPI_OFFLOAD_RDMA=${I_MPI_OFFLOAD_RDMA:-1} I_MPI_OFFLOAD_RDMA_THRESHOLD=${I_MPI_OFFLOAD_RDMA_THRESHOLD:-0}
