#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2015-2023, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8   ****************************************

#[ICS VERSION STRING: unknown]

# PURPOSE:
#
# This file sets FI_* and PSM3_* environment variables which are common
# across all MPIs being used for mpi_apps jobs.
#
# SYNTAX:
#
# This file must be a valid BASH script. In general, anything that's valid
# in BASH is valid here.

# Variables should be specified as:
#	export VARIABLE_NAME=$(VARIABLE_NAME:-variablevalue}
#
# The precedence if a setting is specified in multiple ways is as follows:
# 1. settings added to MPI_CMD_ARGS in the relevant *mpi.params,
#    these will be placed at the end of the mpirun command line.
# 2. variables added to MPI_CMD_ARGS in the relevant *mpi.params with an
#    explicit value such as -genv variable=value
#    without using the ${variable:-value} syntax
# 3. explicit settings of variables in the relevant *mpi.params
#    such as variable=value
#    without using the ${variable:-value} syntax
# 4. variables set in this file with an
#    explicit value such as variable=value
#    without using the ${variable:-value} syntax
# 5. env variables passed into the job such as an export immediately
#    before starting the job or on the job command line.
#    Such as: variable=value ./run_bw5
# 6. env variables set by the system or user.  Such as via .bashrc or
#    explicitly loaded "modules files"
# 7. variables set in this file with the ${variable:-value} syntax
# 8. variables set in the relevant *mpi.paranms file with
#    the ${variable:-value} syntax
# 9. settings in /etc/psm3.conf
# If a variable is specified multiple ways, lowest number above wins.
#
# The logfile created by the job will show relevant env variables as well
# as the mpirun command line.
#
# The ${varname:-value} syntax allows this file to set values typically used
# while still permitting the caller or user to export a different value
# when it's desired to override the value specified in this file.

# SAMPLE Tuning variables:
#
# Uncomment the following lines to enable them.
#

# By default, the run_* scripts use a round-robin method to assign MPI
# processes to hosts. (For example, given 4 hosts, named A, B, C, and D, by
# default 8 processes will be laid out as A, B, C, D, A, B, C, D.) Changing
# PROCS_PER_NODE will change how this is done. For example, PROCS_PER_NODE=2
# results in processes being laid out as A, A, B, B, C, C, D, D.
#export PROCS_PER_NODE=1

# This selects which PSM communications subsystems will be used and the order
# in which they are considered for communications within a job.
# You will not normally need to change this.
#export PSM3_DEVICES=${PSM3_DEVICES:-'self,shm,nic'}
#export PSM3_DEVICES=${PSM3_DEVICES:-'self,nic,shm'}

# When running a GPU application one of these
# three variables must be enabled.  Typically PSM3_GPUDIRECT=1 offers
# better performance, but also depends on the rv kernel module for GPU pinning
#export PSM3_GPUDIRECT=${PSM3_GPUDIRECT:-1}
#export PSM3_ONEAPI_ZE=${PSM3_ONEAPI_ZE:-1}
#export PSM3_CUDA=${PSM3_CUDA:-1}

# for many applications there may be performance advantages to increasing
# the size of the rv kernel module's MR cache
#export PSM3_RV_GPU_CACHE_SIZE=${PSM3_RV_GPU_CACHE_SIZE:-4096}
#export PSM3_RV_MR_CACHE_SIZE=${PSM3_RV_MR_CACHE_SIZE:-8192}

# The PSM RDMA mode controls how data transfers are done.
#export PSM3_RDMA=${PSM3_RDMA:-1}

# PSM3_NIC selects the NIC(s) to consider for use. Selection is by the device
# name (as listed alphabetically in /sys/class/infiniband) or by device number,
# with the first device listed in /sys/class/infiniband being device 0,
# the second device being device 1, and so on. By default,
# PSM3_NIC=any (considers all NICs)
#export PSM3_NIC=${PSM3_NIC:-irdma0}
#export PSM3_NIC=${PSM3_NIC:-2}
#export PSM3_NIC=${PSM3_NIC:-irdma*}

# This controls the NIC selection algorithm when not using PSM Multi-Rail.
# For GPU platforms with PCIe switches GpuRoundRobin will select
# among the NICs closest to the GPU being used by each process.
# RoundRobin is the default and selects NICs based on NUMA locality of the
# CPU being used by each process.
#export PSM3_NIC_SELECTION_ALG=${PSM3_NIC_SELECTION_ALG:-RoundRobin}
#export PSM3_NIC_SELECTION_ALG=${PSM3_NIC_SELECTION_ALG:-GpuRoundRobin}

# These values can enable and control PSM Multi-Rail
# In most cases the default automatic selections will be sufficient.
# Sample for a multi-NIC server with all PSM processes using all NICs
#export PSM3_MULTIRAIL=${PSM3_MULTIRAIL:-1}
# Sample for a multi-NIC server with two NICs explicitly specified for use
# by all processes
#export PSM3_MULTIRAIL=${PSM3_MULTIRAIL:-1} PSM3_MULTIRAIL_MAP=${PSM3_MULTIRAIL_MAP:-irdma0,irdma1}
# Sample for a multi-NIC server with each PSM process using all NUMA local NICs
#export PSM3_MULTIRAIL=${PSM3_MULTIRAIL:-2}
# Sample for a GPU platform with PCIe switches, and for each process using all
# NICs local to its GPU.
#export PSM3_MULTIRAIL=${PSM3_MULTIRAIL:-4}

# If the network is routed so that NICs with different IP subnets can still
# talk to eachother, this causes PSM to treat all IP addresses as part of the
# same network.
#export PSM3_ALLOW_ROUTERS=${PSM3_ALLOW_ROUTERS:-1}

# Use this to explicitly specify a MTU
#export PSM3_MTU=${PSM3_MTU:-4096}

# PSM at job launch can output basic identifying information including it's
# version, location, NIC(s) selected, etc
# The 1st example outputs this for all ranks, the second outputs only for rank 0
#export PSM3_IDENTIFY=${PSM3_IDENTIFY:-1}
#export PSM3_IDENTIFY=${PSM3_IDENTIFY:-'1:'}

# PSM can output performance statistics periodically to a separate file per
# rank.  This controls the frequency of the output (in seconds)
#export PSM3_PRINT_STATS=${PSM3_PRINT_STATS:-1}

# This sets the prefix for the performance statistics files to match the
# output log in the logs/ directory.
export PSM3_PRINT_STATS_PREFIX=${PSM3_PRINT_STATS_PREFIX:-$LOGFILE}

# additional debug output can help when diagnosing NIC or provider
# selection issues.  Use PSM3_TRACEMASK=0x3: to just see 1st rank.
# Use PSM3_TRACEMASK=0x3 to see all ranks.
#export FI_LOG_LEVEL=${FI_LOG_LEVEL:-info}
#export PSM3_TRACEMASK=${PSM3_TRACEMASK:-0x3:}
#export PSM3_TRACEMASK=${PSM3_TRACEMASK:-0x3}

# PSM has the capability to dump its own backtrace on a crash, independent of
# the existing coredump facility. Enabling this feature should cause a
# backtrace to be logged to stderr and to a file in the current working
# directory.
#export PSM3_BACKTRACE=${PSM3_BACKTRACE:-1}

# This variable is used by our caller to force benchmarks to run
# on selected CPU cores.
#export MPI_TASKSET="${MPI_TASKSET:-' -c 1-7'}"
