#
# Copyright (c) 2015 Mellanox, Inc.  All rights reserved.
#
# Sample usage:
# mtt -f ./mtt_ucx_oshmem.ini -v -s ./scratch01
#

#----------------------------------------------------------------------
[MTT]
submit_group_results=1
logfile =  &scratch_root()/mtt_output.log
email = $USER@localhost
drain_timeout=5
min_disk_free=0
terminate_files = &scratch_root()/mtt-stop
mode=fast
pkg=scm
wrapper=oshrun
#wrapper=srun
ofed_shmem_home=&shell("rpm -q  openmpi --queryformat=\'%{prefixes}\' 2>/dev/null")
ofed_shmem_home_version=&shell("@ofed_shmem_home@/bin/ompi_info -V | awk '/Open MPI/ {print $3}'")
mca_global_params=

ompi_scm_url=https://github.com/openucx/ompi-mirror/
test_branch=topic/ucx

ucx_url=https://github.com/uccs/ucx.git

ucx_branch=master
ucx_src_dir=&scratch_root()/ucx_src
ucx_gtest_dir=@ucx_src_dir@/test/gtest
ucx_dir=&get_ini_val(mtt, ucx_src_dir)/install
ucx_lib_dir=&get_ini_val('mtt','ucx_dir')/lib/
ucx_configure=configure-devel

ucx_ports=mlx5_0:1

docommand_timeout_notify_email=@email@
#docommand_timeout_notify_timeout=2:00:00
#docommand_timeout_notify_file=&scratch_root()/mtt-wait
docommand_timeout_backtrace_program=gstack
#docommand_timeout_pernode_before_each_backtrace_program=<<EOF
#echo Extracting sysrq traces from $(hostname) at $(date) ===========
#echo t | sudo tee /proc/sysrq-trigger
#sudo dmesg
#echo ==============================================
#EOF


#break_threshold_timeout_and_fail=35%
save_intermediate_report=1
description=shmem
shuffle_tests=all

ucx_on      = -mca spml ucx
ucx_off     = -mca spml yoda  -mca pml ob1

ucx_tls_def = @ucx_on@
ucx_tls_ud  = @ucx_on@ -x UCX_TLS=ud
ucx_tls_rc  = @ucx_on@ -x UCX_TLS=rc

ucx_tls_by_hca_fast = &enumerate_notempty(\
                      '@ucx_tls_def@',\
                      '@ucx_tls_rc@',\
                      )

ucx_tls_by_hca_long = &enumerate_notempty(\
        '@ucx_tls_ud@',\
        '@ucx_tls_rc@',\
        '@ucx_tls_def@',\
        '@ucx_off@',\
        )

myscript=<<EOT

if [ ! -d "@ucx_lib_dir@" ]; then
    git clone @ucx_url@ @ucx_src_dir@
    cd @ucx_src_dir@
    git checkout @ucx_branch@

    ./autogen.sh
    ./contrib/@ucx_configure@ --prefix=@ucx_dir@ --enable-debug

    make cov
    make -j 9
    make install || echo "UCX install failed" >  &scratch_root()/mtt-stop

fi
EOT

before_mtt_start_exec=<<EOT
rm -f &scratch_root()/mtt-stop
&get_ini_val('mtt','myscript')
EOT

on_exit=<<EOF
EOF

on_kill=<<EOT
echo "This is important echo to process on_kill value as shell command."
&get_ini_val('mtt','on_exit')
EOT

after_mtt_start_exec=<<EOF
echo "This is important echo to process after_mtt_start_exec value as shell command."
&get_ini_val("mtt","on_kill")
EOF

#----------------------------------------------------------------------
[MPI Details: oshmem]

coll_params =  &enumerate_notempty(\
        '', \
        '-mca scoll basic', \
        )

memheap_params_fast =  &enumerate(\
        '', \
        '-mca sshmem mmap', \
        '-mca sshmem verbs', \
        '-mca sshmem sysv', \
        )

params_fast = @memheap_params_fast@ @coll_params@ &get_ini_val(mtt, ucx_tls_by_hca_fast)

#Use separate values during execution:
# fast mode (default)
# long mode

params = @params_fast@

hca_select=-mca sshmem_verbs_hca_name &get_ini_val(mtt, ucx_ports) -mca btl_openib_if_include &get_ini_val(mtt, ucx_ports) -x UCX_NET_DEVICES=&get_ini_val(mtt, ucx_ports)

common = -bind-to core -mca btl_openib_warn_default_gid_prefix 0 -report-bindings -display-map -x SHMEM_SYMMETRIC_HEAP_SIZE=128M &get_ini_val(mtt, mca_global_params) @hca_select@

alloc = --map-by node
alloc_srun = -m cyclic

exec = <<EOF
#!/bin/sh
#SBATCH --job-name=oshmtt --nodes=&getenv('SLURM_NNODES') --ntasks-per-node=&getenv('SLURM_NTASKS_PER_NODE') --partition=&shell('squeue -h -j $SLURM_JOB_ID -o %P') --time=0
#MTT_SCRATCH_DIR=&scratch_root()

export SHMEM_HOME=&test_prefix()
export EXE=&test_executable_abspath()
export EXE_ARGV="&test_argv()"
export NP=&test_np()
export MCA="@common@ @params@ @alloc@ &test_extra_mpi_argv()"
export wrapper="&get_ini_val(mtt, wrapper)"
export ppn="&getenv('SLURM_NTASKS_PER_NODE')"
export nnodes="&getenv('SLURM_NNODES')"
export partition="&shell('squeue -h -j $SLURM_JOB_ID -o %P')"

function srun_env() {
    params_oshrun=( $@ )

    result_srun=""

    for ((i=0; i<${#params_oshrun[@]}; i++)); do
        if [[ "${params_oshrun[$i]}" == "-mca" ]]; then
            result_srun="$result_srun OMPI_MCA_${params_oshrun[((i + 1))]}=${params_oshrun[((i + 2))]}"
            ((i = i + 2))
            continue
        fi
        if [[ "${params_oshrun[$i]}" == "-x" ]]; then
            result_srun="$result_srun ${params_oshrun[((i + 1))]}"
            ((i = i + 1))
            continue
        fi
#        result_srun="$result_srun ${params_oshrun[$i]}"
    done
}

# if running under mtt - save script into scratch dir
if [ -n "$MTT_TEST_SID" ]; then
    echo MTT_TEST_SID=&scratch_root()/run_test_${MTT_TEST_SID}.sbatch
    cp $0 &scratch_root()/run_test_${MTT_TEST_SID}.sbatch
fi

ulimit -c unlimited

if [[ "$wrapper" == "oshrun" ]]; then
    set -x
    date
    $SHMEM_HOME/bin/oshrun -np $NP $MCA $EXE $EXE_ARGV
else
    srun_env $MCA
    #for var in $result_srun; do
    #    export $var
    #done

    # srun can be used w/o params here - as it will read SLURM_ vars from sbatch header
    set -x
    date
    env $result_srun srun --cpu_bind=core @alloc_srun@ --mpi=pmi2 --nodes=$nnodes --tasks-per-node $ppn -p $partition $EXE $EXE_ARGV
fi
rc=$?
exit $rc

EOF

exec:simple      =<<EOF
#!/bin/bash
#SBATCH --job-name=oshmem_simple --time=00:30:00
#SBATCH -w &getenv('SLURM_NODELIST')
#SBATCH --nodes=&getenv('SLURM_NNODES') --ntasks-per-node=&getenv('SLURM_NTASKS_PER_NODE') --partition=&shell('squeue -h -j $SLURM_JOB_ID -o %P')

export EXE=&test_executable_abspath()

set -x
$EXE &test_argv()
EOF

#----------------------------------------------------------------------
[MPI get: oshmem_ofed]
mpi_details = oshmem
module = AlreadyInstalled
alreadyinstalled_dir=&get_ini_val('mtt', 'ofed_shmem_home')
alreadyinstalled_mpi_type=OMPI
skip_section=&mtt_switch(&get_ini_val('mtt', 'pkg'), 'ofed', 0, 'default', 1)


[MPI install: oshmem_ofed]
mpi_get = oshmem_ofed
module = Analyze::OMPI
description = Already installed MPI
product_install_dir=&get_ini_val('mtt', 'ofed_shmem_home')
product_version=&get_ini_val('mtt', 'ofed_shmem_home_version')
skip_section=&mtt_switch(&get_ini_val('mtt', 'pkg'), 'ofed', 0, 'default', 1)

#----------------------------------------------------------------------
[MPI get: oshmem_scm_test]
mpi_details = oshmem
module = SCM
scm_module = Git
scm_url = &get_ini_val(mtt, ompi_scm_url)
scm_rev = &get_ini_val(mtt, test_branch)
scm_clone_all = 1
skip_section=&mtt_switch(&get_ini_val('mtt', 'pkg'), 'ofed', 1, 'default', 0)
scm_post_copy = <<EOT
EOT


[mpi_install_scm_common]
skip_section=&mtt_switch(&get_ini_val('mtt','pkg'), 'default', 0, 'ofed', 1)
module = OMPI
setenv = mellanox_autodetect    yes
ompi_make_all_arguments = -j &get_processor_count()
ompi_autogen=1
save_stdout_on_success = 1
merge_stdout_stderr = 1
#ompi_make_check = 1
#ompi_vpath_mode = none
product_install_dir=&get_mpi_install_dir()
product_version=&get_ini_val('mtt', 'ofed_shmem_home_version')
ompi_compiler_name = gnu
ompi_configure_arguments = --with-platform=contrib/platform/mellanox/optimized --with-ucx=&get_ini_val('mtt','ucx_dir')

#----------------------------------------------------------------------

[MPI Install: oshmem_scm_module_debug]
mpi_get = oshmem_scm_test
setenv = mellanox_debug         yes
include_section = mpi_install_scm_common
skip_section=&mtt_switch(&get_ini_val('mtt', 'pkg'), 'ofed', 1, 'default', 0)

#----------------------------------------------------------------------

[Test get: ucx_perftest]
module = SCM
scm_module = Git
scm_url= https://github.com/uccs/ucx.git
scm_clone_all = 1
scm_post_copy = <<EOT
./autogen.sh
EOT

[Test build: ucx_perftest]
test_get = ucx_perftest
module = Shell
shell_build_command = <<EOT
./contrib/configure-devel --prefix=$PWD/install --enable-debug --with-mpi=&test_prefix()
make -j 9
make install
EOT

#----------------------------------------------------------------------

[Test get: osu_micro_benchmark]
module = Download
download_url = http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-4.4.1.tar.gz

[Test build: osu_micro_benchmark]
test_get = osu_micro_benchmark
save_stdout_on_success = 1
merge_stdout_stderr = 1
module = Shell
shell_build_command = <<EOT
export LDFLAGS="-loshmem"
aclocal
./configure --prefix=$PWD/bin/ CC=&test_prefix()/bin/mpicc
make
make install
EOT

#----------------------------------------------------------------------

[Test get: tests-mellanox]
module = SCM
scm_module = Git
scm_url = https://github.com/openshmem-org/tests-mellanox.git
scm_clone_all = 1

[Test build: verifier]
test_get = tests-mellanox
save_stdout_on_success = 1
merge_stdout_stderr = 1
module = Shell

shell_build_command = <<EOT
cd verifier
export SHMEM_HOME=&test_prefix_pretty()
export CC=''
./autogen.sh
./configure --prefix=$PWD/install
make -j9 install

EOT

#----------------------------------------------------------------------

[Test run]
pass = &and(&cmd_wifexited(), &eq(&cmd_wexitstatus(), 0))
specify_module = Simple
save_stdout_on_pass = 1
timeout = 25m
stdout_save_lines = -1
merge_stdout_stderr = 1
np = &env_max_procs()

[Test run: verifier]
include_section = Test run
test_build = verifier

simple_first:tests =  &enumerate("verifier/install/bin/oshmem_test")
simple_first:argv = &enumerate(\
						"exec --no-colour --task=basic",\
						"exec --no-colour --task=coll",\
						"exec --no-colour --task=data",\
						"exec --no-colour --task=atomic",\
						"exec --no-colour --task=lock",\
						"exec --no-colour --task=reduce",\
						"exec --no-colour --task=sync",\
						"exec --no-colour --task=stride",\
						"exec --no-colour --task=mix",\
						"exec --no-colour --task=misc",\
						)

[run_osu_micro_benchmark_common]
skip_section=&mtt_switch(&get_ini_val('mtt','pkg'), 'default', 0)
include_section = Test run
test_build = osu_micro_benchmark
skipped = 0
simple_first:argv = &enumerate("heap","global")
np =  &env_max_procs()
timeout = 10m

[Test run: osu_micro_benchmark_oshm_put]
include_section = run_osu_micro_benchmark_common
simple_first:tests = &enumerate(&find_executables("openshmem/osu_oshm_put"))
np=2

[Test run: osu_micro_benchmark_oshm_get]
include_section = run_osu_micro_benchmark_common
simple_first:tests = &enumerate(&find_executables("openshmem/osu_oshm_get"))
np=2

[Test run: osu_micro_benchmark_oshm_put_mr]
include_section = run_osu_micro_benchmark_common
simple_first:tests = &enumerate(&find_executables("openshmem/osu_oshm_put_mr"))

[Test run: osu_micro_benchmark_oshm_atomics]
include_section = run_osu_micro_benchmark_common
simple_first:tests = &enumerate(&find_executables("openshmem/osu_oshm_atomics"))

[Test run: ucx_perftest]
include_section = Test run
test_build = ucx_perftest
simple_first:tests = install/bin/ucx_perftest
argv = -d &get_ini_val(mtt, ucx_ports) -x &enumerate(ud,rc) -t &enumerate(am_lat)
np=2

#======================================================================
# Reporter phase
#======================================================================

[Reporter: HTML_ONLY]
module = HTMLReport
email_to = &get_ini_val('mtt', 'email')
email_subject = MTT finished for @INI_BASENAME@ on &shell("hostname"), $overall_mtt_status, pkg is &get_ini_val(mtt, pkg), mode &get_ini_val(mtt, mode)
header=<<EOF
<ul>
<li>slurm jobid is &getenv('SLURM_JOBID')</li>
</ul>
EOF