
properties([disableConcurrentBuilds(abortPrevious: true)])
def DO_RUN=1
def TARGET="main"

def check_target() {
  echo "CHANGE_TARGET = ${env.CHANGE_TARET}"
  if (changeRequest()) {
    TARGET = env.CHANGE_TARGET
  }
}

def skip() {
  def file = "${env.WORKSPACE}/commit_id"
  if (!fileExists(file)) {
    echo "CI Run has not rebased with ofiwg/libfabric. Please Rebase."
    return 1
  }

  def changes = readFile file
  def changeStrings = new ArrayList<String>()

  for (line in changes.readLines()) {
    changeStrings.add(line)
  }

  echo "${changeStrings.toArray()}"
  if (changeStrings.toArray().every { it =~ /(?:fabtests\/pytests|man|prov\/efa|prov\/opx).*$/ }) {
    echo "DONT RUN!"
    return 0
  }

  if (changeStrings.isEmpty()) {
    echo "DONT RUN!"
    return 0
  }

  return 1
}

pipeline {
  agent { node { label 'master' } }
  options {
      timestamps()
      timeout(activity: true, time: 1, unit: 'HOURS')
  }
  environment {
      JOB_CADENCE = 'PR'
  }

  stages {
    stage ('opt-out') {
      steps {
        script {
          check_target()
        }
        withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
          sh """
           mkdir ${env.WORKSPACE}/py_scripts
            git clone ${env.UPSTREAM} ${env.WORKSPACE}/py_scripts
            ${env.SKIP_PATH}/skip.sh ${env.WORKSPACE} ${TARGET}
          """
        }
        script {
          DO_RUN=skip()
        }
      }
    }
    stage ('build') {
      when { equals expected: 1, actual: DO_RUN }
      steps {
        withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
          sh """
            env
            (
              echo "-----------------------------------------------------"
              echo "Copy build dirs."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=builddir
              echo "Copy build dirs completed."
              echo "-----------------------------------------------------"

              echo "-----------------------------------------------------"
              echo "Copy log dirs."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=logdir
              echo "Copy log dirs completed."
              echo "-----------------------------------------------------" 

              echo "-----------------------------------------------------"
              echo "Building libfabric reg."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=libfabric
              echo "-----------------------------------------------------"
              echo "Building libfabric dbg."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=libfabric --ofi_build_mode=dbg
              echo "-----------------------------------------------------"
              echo "Building libfabric dl."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=libfabric --ofi_build_mode=dl
              echo "Libfabric builds completed."

              echo "-----------------------------------------------------"
              echo "Building fabtests reg."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=fabtests
              echo "-----------------------------------------------------"
              echo "Building fabtests dbg."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=fabtests --ofi_build_mode=dbg
              echo "-----------------------------------------------------"
              echo "Building fabtests dl."
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/build.py --build_item=fabtests --ofi_build_mode=dl
              echo 'Fabtests builds completed.'
            )
          """
        }
      }
    }
    stage('parallel-tests') {
        when { equals expected: 1, actual: DO_RUN }
        parallel {
          stage('MPI_verbs-rxm') {
            agent {node {label 'mlx5'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=verbs --util=rxm --test=IMB --imb_grp=1
                    echo "IMB verbs-rxm Group 1 completed."
                    python3.7 runtests.py --prov=verbs --util=rxm --test=IMB --imb_grp=2
                    echo "IMB verbs-rxm Group 2 completed."
                    python3.7 runtests.py --prov=verbs --util=rxm --test=IMB --imb_grp=3
                    echo "IMB verbs-rxm Group 3 completed."
                    python3.7 runtests.py --prov=verbs --util=rxm --test=osu
                    echo "OSU verbs-rxm completed."
                    echo "MPI-verbs-rxm completed."
                  )
                """
              }
            }
          }
          stage('MPI_tcp-rxm-2') {
            agent {node {label 'cvl'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=tcp --util=rxm --test=IMB --imb_grp=2
                    echo "MPI-tcp-rxm-2 completed."
                  )
                """
              }
            }
          }
          stage('tcp') {
            agent {node {label 'cvl'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=tcp --test=fabtests
                    python3.7 runtests.py --prov=tcp --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=tcp --test=fabtests --ofi_build_mode=dl
                    echo "tcp completed."
                  )
                """
              }
            }
          }
          stage('verbs-rxm') {
            agent {node {label 'mlx5'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=verbs --util=rxm --test=fabtests
                    python3.7 runtests.py --prov=verbs --util=rxm --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=verbs --util=rxm --test=fabtests --ofi_build_mode=dl
                    echo "verbs-rxm completed."
                  )
                """
              }
            }
          }
          stage('verbs-rxd') {
            agent {node {label 'mlx5 && edr'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=verbs --util=rxd --test=fabtests
                    python3.7 runtests.py --prov=verbs --util=rxd --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=verbs --util=rxd --test=fabtests --ofi_build_mode=dl
                    echo "verbs-rxd completed."
                  )
                """
              }
            }
          }
          stage('udp') {
            agent {node {label 'cvl'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=udp --test=fabtests
                    python3.7 runtests.py --prov=udp --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=udp --test=fabtests --ofi_build_mode=dl
                    echo "udp completed."
                  )
                """
              }
            }
          }
          stage('shm') {
            agent {node {label 'cvl'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=shm --test=fabtests
                    python3.7 runtests.py --prov=shm --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=shm --test=fabtests --ofi_build_mode=dl
                    echo "shm completed."
                  )
                """
              }
            }
          }
          stage('sockets') {
            agent {node {label 'cvl'}}
            options { skipDefaultCheckout() }
            steps {
              withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
                sh """
                  env
                  (
                    cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                    python3.7 runtests.py --prov=sockets --test=fabtests
                    python3.7 runtests.py --prov=sockets --test=fabtests --ofi_build_mode=dbg
                    python3.7 runtests.py --prov=sockets --test=fabtests --ofi_build_mode=dl
                    echo "sockets completed."
                  )
                """
              }
            }
          }
          stage('psm3') {
          agent {node {label 'mlx5 && edr'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  export PSM3_IDENTIFY=1
                  export FI_LOG_LEVEL=info
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=psm3 --test=fabtests
                  python3.7 runtests.py --prov=psm3 --test=fabtests --ofi_build_mode=dbg
                  python3.7 runtests.py --prov=psm3 --test=fabtests --ofi_build_mode=dl
                  echo "psm3 completed."
                )
              """
            }
          }
        }
        stage('MPI_tcp-rxm-1') {
          agent {node {label 'cvl'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=tcp --util=rxm --test=IMB --imb_grp=1
                  echo "MPI-tcp-rxm-1 completed."
                )
              """
            }
          }
        }
        stage('MPI_tcp-rxm-3') {
          agent {node {label 'cvl'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=tcp --util=rxm --test=IMB --imb_grp=3
                  echo "MPI-tcp-rxm-3 completed."
                  python3.7 runtests.py --prov=tcp --util=rxm --test=osu
                  echo "OSU verbs-rxm completed."
                )
              """
            }
          }
        }
        stage('MPICH testsuite') {
          agent {node {label 'mlx5'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=verbs --util=rxm --test=mpichtestsuite
                  echo "verbs-rxm MPICH testsuite completed."
                  python3.7 runtests.py --prov=tcp --util=rxm --test=mpichtestsuite
                  echo "tcp-rxm MPICH testsuite completed."
                  python3.7 runtests.py --prov=sockets --test=mpichtestsuite
                  echo "sockets MPICH testsuite completed."
                  echo "MPICH testsuite completed."
                )
              """
            }
          }
        }
        stage('SHMEM') {
          agent {node {label 'mlx5'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=tcp --test=shmem
                  echo "SHMEM tcp completed."
                  python3.7 runtests.py --prov=verbs --test=shmem
                  echo "SHMEM verbs completed."
                  python3.7 runtests.py --prov=sockets --test=shmem
                  echo "SHMEM sockets completed."
                  echo "SHMEM completed."
                )
              """
            }
          }
        }
        stage('oneCCL') {
          agent {node {label 'cvl'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=tcp --util=rxm --test=oneccl
                  echo "oneCCL tcp-rxm completed."
                  python3.7 runtests.py --prov=psm3 --test=oneccl
                  echo "oneCCL psm3 completed."
                  echo "OneCCL completed."
                )
              """
            }
          }
        }
        stage('oneCCL-GPU') {
          agent {node {label 'ze'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=tcp --test=onecclgpu
                  echo "oneCCL-GPU completed."
                )
              """
            }
          }
        }
        stage('ze-shm') {
          agent {node {label 'ze'}}
          options { skipDefaultCheckout() }
          steps {
            withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
              sh """
                env
                (
                  cd /opt${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/
                  python3.7 runtests.py --prov=shm --device='ze'
                  echo "ze-shm completed."
                )
              """
            }
          }
        }
      }
    }
    stage ('Summary') {
      when { equals expected: 1, actual: DO_RUN }
      steps {
        withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) {
          sh """
            env
            (
              python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/summary.py --summary_item=all
            )
          """
        }
      }
    }
  }

  post {
    always {
      withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) {
        sh "python3.7 ${env.WORKSPACE}/py_scripts/contrib/intel/jenkins/summary.py --summary_item=all"
      }
    }
    cleanup {
      withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) {
        dir ("${env.CI_INSTALL_DIR}/${env.JOB_NAME}/${env.BUILD_NUMBER}"){
          deleteDir()
        }
        dir("${env.WORKSPACE}") {
          deleteDir()
        }
        dir("${env.WORKSPACE}@tmp") {
          deleteDir()
        }
      }
    }
  }
}

