#!/bin/bash
# BEGIN_ICS_COPYRIGHT8 ****************************************
#
# Copyright (c) 2023, Intel Corporation
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of Intel Corporation nor the names of its contributors
#       may be used to endorse or promote products derived from this software
#       without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# END_ICS_COPYRIGHT8   ****************************************

#[ICS VERSION STRING: unknown]

# PURPOSE:
#
# This file sets environment variables for a oneCCL job.
# Note that there are many such variables.
#
# SYNTAX:
#
# This file must be a valid BASH script. In general, anything that's valid
# in BASH is valid here.

# To pass variables to oneCCL, they should be specified as:
#	export VARIABLE_NAME=$(VARIABLE_NAME:-variablevalue}
#
# any other settings must be specified as:
#	export MPI_CMD_ARGS="$MPI_CMD_ARGS <settings>"
#
# The precedence if a setting is specified in multiple ways is as follows:
# 1. settings added to MPI_CMD_ARGS in this file
#    these will be placed at the end of the mpirun command line.
# 2. settings added to MPI_CMD_ARGS in intelmpi.params
#    these will be placed at the end of the mpirun command line before any
#    from this file
# 3. variables added to MPI_CMD_ARGS in this file with an
#    explicit value such as -genv variable=value
#    without using the ${variable:-value} syntax
# 4. variables added to MPI_CMD_ARGS in intelmpi.params with an
#    explicit value such as -genv variable=value
#    without using the ${variable:-value} syntax
# 5. explicit settings of variables in this file
#    such as variable=value
#    without using the ${variable:-value} syntax
# 6. variables set in the intelmpi.params file with an
#    explicit value such as variable=value
#    without using the ${variable:-value} syntax
# 7. variables set in the psm3.params file with an
#    explicit value such as variable=value
#    without using the ${variable:-value} syntax
# 8. env variables passed into the job such as an export immediately
#    before starting the job or on the job command line.
#    Such as: variable=value ./run_oneccl
# 9. env variables set by the system or user.  Such as via .bashrc or
#    explicitly loaded "modules files"
# 10. variables set in the psm3.params file with the ${variable:-value} syntax
# 11. variables set in intelmpi.params with the ${variable:-value} syntax
# 12. variables set in this file with the ${variable:-value} syntax
# 13. settings in /etc/psm3.conf
# If a variable is specified multiple ways, lowest number above wins.
#
# The logfile created by the job will show relevant env variables as well
# as the mpirun command line.
#
# The ${varname:-value} syntax allows this file to set values typically used
# while still permitting the caller or user to export a different value
# when it's desired to override the value specified in this file.

# SAMPLE Tuning variables:
#
# Uncomment the following lines to enable them.
#

# Path to oneCCL library
export ONECCL_DIR=${ONECCL_DIR:-$MPICH_PREFIX/../../ccl/latest/}

# Path to oneCCL example programs, such as "benchmark"
export ONECCL_EXAMPLES_DIR=${ONECCL_EXAMPLES_DIR:-$ONECCL_DIR/examples}

# Should a CPU or GPU enabled version of oneCCL be used
#export ccl_configuraton=${ccl_configuration:-cpu}
#export ccl_configuraton=${ccl_configuration:-cpu_gpu_dpcpp}

# Select whether oneCCL should use mpi interface or directly use OFI
export CCL_ATL_TRANSPORT=${CCL_ATL_TRANSPORT:-mpi}
#export CCL_ATL_TRANSPORT=${CCL_ATL_TRANSPORT:-ofi}

# Will oneCCL consider only NUMA "local" NICs or any NIC ("global")
export CCL_MNIC=${CCL_MNIC:-local}
#export CCL_MNIC=${CCL_MNIC:-global}

# can select NICs explicitly when using OFI directly
#export CCL_MNIC_NAME=${CCL_MNIC_NAME:-irdma0,irdma1,irdma2,irdma3}
#export CCL_MNIC_COUNT=${CCL_MNIC_COUNT:-4}

# Now many oneCCL worker threads per process
#export CCL_WORKER_COUNT=${CCL_WORKER_COUNT:-8}

# Which CPU cores should be used for workers
# must specify CCL_WORK_COUNT * processes per node
#export CCL_WORKER_AFFINITY=${CCL_WORKER_AFFINITY:-38,39,40,41,42,43,44,45,88,89,90,91,92,93,94,95}

# may want to tell Intel MPI launcher where to pin processes
#export I_MPI_PIN_PROCESSOR_LIST=${I_MPI_PIN_PROCESSOR_LIST:-1,57}

# AllReduce algorithm to use
#export CCL_ALLREDUCE=${CCL_ALLREDUCE:-ring}
#export CCL_ALLREDUCE=${CCL_ALLREDUCE:-rabenseifner}

# Enable debug output
#export CCL_LOG_LEVEL=${CCL_LOG_LEVEL:-info}
#export CCL_LOG_LEVEL=${CCL_LOG_LEVEL:-debug}
