# Copyright (c) 2010-2023, Lawrence Livermore National Security, LLC. Produced
# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
# LICENSE and NOTICE for details. LLNL-CODE-806117.
#
# This file is part of the MFEM library. For more information and source code
# availability visit https://mfem.org.
#
# MFEM is free software; you can redistribute it and/or modify it under the
# terms of the BSD-3 license. We welcome feedback and contributions, see file
# CONTRIBUTING.md for details.

MFEM_DIR ?= ../..
MFEM_BUILD_DIR ?= ../..
SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/tests/unit/,)
CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk

MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)

CC = $(MFEM_CXX)
CCOPTS = -g
CCC = $(CC) $(CCOPTS)

# -I$(MFEM_DIR) is needed by some tests, e.g. to #include "general/text.hpp"
INCLUDES = -I$(or $(SRC:%/=%),.) -I$(MFEM_DIR)

SOURCE_FILES := $(sort $(wildcard $(SRC)*/*.cpp))
CEED_SOURCE_FILES = $(sort $(wildcard $(SRC)ceed/*.cpp))
MINI_SOURCE_FILES = $(sort $(wildcard $(SRC)miniapps/*.cpp))
SOURCE_FILES := $(filter-out \
   $(CEED_SOURCE_FILES) $(MINI_SOURCE_FILES), $(SOURCE_FILES))
HEADER_FILES = $(SRC)catch.hpp $(SRC)unit_tests.hpp
OBJECT_FILES = $(SOURCE_FILES:$(SRC)%.cpp=%.o)
DATA_DIR = data

SEQ_MAIN_OBJ = unit_test_main.o
PAR_MAIN_OBJ = punit_test_main.o
CUDA_MAIN_OBJ = cunit_test_main.o
PCUDA_MAIN_OBJ = pcunit_test_main.o

# Sedov numerical seq/par files and tests
SEDOV_FILES = $(SRC)miniapps/test_sedov.cpp

USE_CUDA := $(MFEM_USE_CUDA:NO=)
SEQ_SEDOV_TESTS = sedov_tests_cpu sedov_tests_debug
SEQ_SEDOV_TESTS += $(if $(USE_CUDA),sedov_tests_cuda)
SEQ_SEDOV_TESTS += $(if $(USE_CUDA),sedov_tests_cuda_uvm)
PAR_SEDOV_TESTS = $(SEQ_SEDOV_TESTS:%=p%)

SEQ_SEDOV_CPU_OBJ_FILES = $(SEDOV_FILES:$(SRC)%.cpp=%.cpu.o)
SEQ_SEDOV_DEBUG_OBJ_FILES = $(SEDOV_FILES:$(SRC)%.cpp=%.debug.o)
SEQ_SEDOV_CUDA_OBJ_FILES = $(if $(USE_CUDA),$(SEDOV_FILES:$(SRC)%.cpp=%.cuda.o))
SEQ_SEDOV_CUDA_UVM_OBJ_FILES = $(if $(USE_CUDA),$(SEDOV_FILES:$(SRC)%.cpp=%.cuda_uvm.o))

PAR_SEDOV_CPU_OBJ_FILES = $(SEDOV_FILES:$(SRC)%.cpp=%.pcpu.o)
PAR_SEDOV_DEBUG_OBJ_FILES = $(SEDOV_FILES:$(SRC)%.cpp=%.pdebug.o)
PAR_SEDOV_CUDA_OBJ_FILES = $(if $(USE_CUDA),$(SEDOV_FILES:$(SRC)%.cpp=%.pcuda.o))
PAR_SEDOV_CUDA_UVM_OBJ_FILES = $(if $(USE_CUDA),$(SEDOV_FILES:$(SRC)%.cpp=%.pcuda_uvm.o))

# TMOP numerical seq/par files and tests
TMOP_FILES = $(SRC)miniapps/test_tmop_pa.cpp

SEQ_TMOP_TESTS = tmop_pa_tests_cpu tmop_pa_tests_debug
SEQ_TMOP_TESTS += $(if $(USE_CUDA),tmop_pa_tests_cuda)
# SEQ_TMOP_TESTS += $(if $(USE_CUDA),tmop_tests_cuda_uvm)
PAR_TMOP_TESTS = ptmop_pa_tests_cpu
PAR_TMOP_TESTS += $(if $(USE_CUDA),ptmop_pa_tests_cuda)

SEQ_TMOP_CPU_OBJ_FILES = $(TMOP_FILES:$(SRC)%.cpp=%.cpu.o)
SEQ_TMOP_DEBUG_OBJ_FILES = $(TMOP_FILES:$(SRC)%.cpp=%.debug.o)
SEQ_TMOP_CUDA_OBJ_FILES = $(if $(USE_CUDA),$(TMOP_FILES:$(SRC)%.cpp=%.cuda.o))
SEQ_TMOP_CUDA_UVM_OBJ_FILES = $(if $(USE_CUDA),$(TMOP_FILES:$(SRC)%.cpp=%.cuda_uvm.o))

PAR_TMOP_CPU_OBJ_FILES = $(TMOP_FILES:$(SRC)%.cpp=%.pcpu.o)
PAR_TMOP_DEBUG_OBJ_FILES = $(TMOP_FILES:$(SRC)%.cpp=%.pdebug.o)
PAR_TMOP_CUDA_OBJ_FILES = $(if $(USE_CUDA),$(TMOP_FILES:$(SRC)%.cpp=%.pcuda.o))
PAR_TMOP_CUDA_UVM_OBJ_FILES = $(if $(USE_CUDA),$(TMOP_FILES:$(SRC)%.cpp=%.pcuda_uvm.o))

# seq/par files and tests
SEQ_UNIT_TESTS = unit_tests $(if $(USE_CUDA),cunit_tests)
SEQ_UNIT_TESTS += $(SEQ_SEDOV_TESTS) $(SEQ_TMOP_TESTS)
PAR_UNIT_TESTS = punit_tests $(if $(USE_CUDA),pcunit_tests)
PAR_UNIT_TESTS += $(PAR_SEDOV_TESTS) $(PAR_TMOP_TESTS)

# Ceed tests
USE_CEED := $(MFEM_USE_CEED:NO=)
CEED_OBJ = $(CEED_SOURCE_FILES:$(SRC)%.cpp=%.o)
SEQ_UNIT_TESTS += $(if $(USE_CEED),ceed_tests)

# Debug device tests
DEBUG_DEVICE_FILE = $(SRC)miniapps/test_debug_device.cpp
DEBUG_DEVICE_OBJ = $(DEBUG_DEVICE_FILE:$(SRC)%.cpp=%.o)
DEBUG_DEVICE_TEST = debug_device_tests
SEQ_UNIT_TESTS += $(DEBUG_DEVICE_TEST)

ifeq ($(MFEM_USE_MPI),NO)
   UNIT_TESTS = $(SEQ_UNIT_TESTS)
else
   UNIT_TESTS = $(PAR_UNIT_TESTS) $(SEQ_UNIT_TESTS)
endif

all: $(UNIT_TESTS)

.SUFFIXES:
.SUFFIXES: .cpp .o
.PHONY: all clean

unit_tests: $(SEQ_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LIB_FILE) $(CONFIG_MK) $(DATA_DIR)
	$(CCC) $(SEQ_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LINK_FLAGS) $(MFEM_LIBS) -o $(@)

punit_tests: $(PAR_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LIB_FILE) $(CONFIG_MK) $(DATA_DIR)
	$(CCC) $(PAR_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LINK_FLAGS) $(MFEM_LIBS) -o $(@)

cunit_tests: $(CUDA_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LIB_FILE) $(CONFIG_MK) $(DATA_DIR)
	$(CCC) $(CUDA_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LINK_FLAGS) $(MFEM_LIBS) -o $(@)

pcunit_tests: $(PCUDA_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LIB_FILE) $(CONFIG_MK) $(DATA_DIR)
	$(CCC) $(PCUDA_MAIN_OBJ) $(OBJECT_FILES) $(MFEM_LINK_FLAGS) $(MFEM_LIBS) -o $(@)

ceed_tests: $(CEED_OBJ) $(MFEM_LIB_FILE) $(CONFIG_MK) $(DATA_DIR)
	$(CCC) $(CEED_OBJ) $(MFEM_LINK_FLAGS) $(MFEM_LIBS) -o $(@)

$(DEBUG_DEVICE_TEST): $(DEBUG_DEVICE_OBJ) $(MFEM_LIB_FILE) \
 $(CONFIG_MK)
	$(CCC) $(DEBUG_DEVICE_OBJ) $(MFEM_LINK_FLAGS) \
	   $(MFEM_LIBS) -o $(@)

# Note: in this rule, we always use the full path to the source file as a
# workaround for an issue with coveralls.
$(OBJECT_FILES) $(SEQ_MAIN_OBJ) $(PAR_MAIN_OBJ) $(CUDA_MAIN_OBJ) \
 $(PCUDA_MAIN_OBJ) $(DEBUG_DEVICE_OBJ): %.o: $(SRC)%.cpp $(HEADER_FILES) \
 $(CONFIG_MK)
	@mkdir -p $(@D)
	$(CCC) $(MFEM_FLAGS) $(INCLUDES) -c $(abspath $(<)) -o $(@)

$(CEED_OBJ): %.o: $(SRC)%.cpp $(HEADER_FILES) $(CONFIG_MK)
	@mkdir -p $(@D)
	$(CCC) $(MFEM_FLAGS) $(INCLUDES) -c $(abspath $(<)) -o $(@)

# Template rule for devices
# SEDOV Template rule for devices
# 1: postfix of 'sedov_tests_', 2: prefix of '_OBJECT_FILES',
# 3: configuration string for the MFEM device, separated with '$(comma)'
comma=,
define sedov_tests
sedov_tests_$(1): SEDOV_TESTS_FLAGS=-DMFEM_SEDOV_DEVICE='"$(3)"'
sedov_tests_$(1): unit_test_main.o $$(SEQ_SEDOV_$(2)_OBJ_FILES) \
 $$(MFEM_LIB_FILE) $$(CONFIG_MK) $$(DATA_DIR)
	$$(CCC) $$(<) $$(SEQ_SEDOV_$(2)_OBJ_FILES) \
	   $$(MFEM_LINK_FLAGS) $$(MFEM_LIBS) -o $$(@)
$$(SEQ_SEDOV_$(2)_OBJ_FILES): %.$(1).o: $$(SRC)%.cpp $$(HEADER_FILES) $$(CONFIG_MK)
	@mkdir -p $$(@D)
	$$(CCC) $$(MFEM_FLAGS) $$(INCLUDES) -c $$(abspath $$(<)) \
	   $$(SEDOV_TESTS_FLAGS) -o $$(@)
endef
$(eval $(call sedov_tests,cpu,CPU,cpu))
$(eval $(call sedov_tests,debug,DEBUG,debug))
$(eval $(call sedov_tests,cuda,CUDA,cuda))
$(eval $(call sedov_tests,cuda_uvm,CUDA_UVM,cuda:uvm))

define psedov_tests
psedov_tests_$(1): SEDOV_TESTS_FLAGS =-DMFEM_SEDOV_MPI
psedov_tests_$(1): SEDOV_TESTS_FLAGS+=-DMFEM_SEDOV_DEVICE='"$(3)"'
psedov_tests_$(1): punit_test_main.o $$(PAR_SEDOV_$(2)_OBJ_FILES) \
 $$(MFEM_LIB_FILE) $$(CONFIG_MK) $$(DATA_DIR)
	$$(CCC) $$(<) $$(PAR_SEDOV_$(2)_OBJ_FILES) \
	   $$(MFEM_LINK_FLAGS) $$(MFEM_LIBS) -o $$(@)
$$(PAR_SEDOV_$(2)_OBJ_FILES): %.p$(1).o: $$(SRC)%.cpp $$(HEADER_FILES) $$(CONFIG_MK)
	@mkdir -p $$(@D)
	$$(CCC) $$(MFEM_FLAGS) $$(INCLUDES) -c $$(abspath $$(<)) \
	   $$(SEDOV_TESTS_FLAGS) -o $$(@)
endef
$(eval $(call psedov_tests,cpu,CPU,cpu))
$(eval $(call psedov_tests,debug,DEBUG,debug))
$(eval $(call psedov_tests,cuda,CUDA,cuda))
$(eval $(call psedov_tests,cuda_uvm,CUDA_UVM,cuda:uvm))

# For out-of-source builds, copy the meshes in ../../miniapps/meshing from the
# source location; these are used by the TMOP tests.
.PHONY: copy-miniapps-meshing-data
copy-miniapps-meshing-data:
	$(MAKE) -C ../../miniapps/meshing copy-data

# TMOP Template rule for devices
# 1: postfix of 'tmop_tests_', 2: prefix of '_OBJECT_FILES',
# 3: configuration string for the MFEM device, separated with '$(comma)'
define tmop_pa_tests
tmop_pa_tests_$(1): | $(if $(SRC),copy-miniapps-meshing-data)
tmop_pa_tests_$(1): TMOP_TESTS_FLAGS=-DMFEM_TMOP_DEVICE='"$(3)"'
tmop_pa_tests_$(1): unit_test_main.o $$(SEQ_TMOP_$(2)_OBJ_FILES) \
 $$(MFEM_LIB_FILE) $$(CONFIG_MK) $$(DATA_DIR)
	$$(CCC) $$(<) $$(SEQ_TMOP_$(2)_OBJ_FILES) \
	   $$(MFEM_LINK_FLAGS) $$(MFEM_LIBS) -o $$(@)
$$(SEQ_TMOP_$(2)_OBJ_FILES): %.$(1).o: $$(SRC)%.cpp $$(HEADER_FILES) $$(CONFIG_MK)
	@mkdir -p $$(@D)
	$$(CCC) $$(MFEM_FLAGS) $$(INCLUDES) -c $$(abspath $$(<)) \
	   $$(TMOP_TESTS_FLAGS) -o $$(@)
endef
$(eval $(call tmop_pa_tests,cpu,CPU,cpu))
$(eval $(call tmop_pa_tests,debug,DEBUG,debug))
$(eval $(call tmop_pa_tests,cuda,CUDA,cuda))
# $(eval $(call tmop_pa_tests,cuda_uvm,CUDA_UVM,cuda:uvm))

define ptmop_pa_tests
ptmop_pa_tests_$(1): | $(if $(SRC),copy-miniapps-meshing-data)
ptmop_pa_tests_$(1): TMOP_TESTS_FLAGS =-DMFEM_TMOP_MPI
ptmop_pa_tests_$(1): TMOP_TESTS_FLAGS+=-DMFEM_TMOP_DEVICE='"$(3)"'
ptmop_pa_tests_$(1): punit_test_main.o $$(PAR_TMOP_$(2)_OBJ_FILES) \
 $$(MFEM_LIB_FILE) $$(CONFIG_MK) $$(DATA_DIR)
	$$(CCC) $$(<) $$(PAR_TMOP_$(2)_OBJ_FILES) \
	   $$(MFEM_LINK_FLAGS) $$(MFEM_LIBS) -o $$(@)
$$(PAR_TMOP_$(2)_OBJ_FILES): %.p$(1).o: $$(SRC)%.cpp $$(HEADER_FILES) $$(CONFIG_MK)
	@mkdir -p $$(@D)
	$$(CCC) $$(MFEM_FLAGS) $$(INCLUDES) -c $$(abspath $$(<)) \
	   $$(TMOP_TESTS_FLAGS) -o $$(@)
endef
$(eval $(call ptmop_pa_tests,cpu,CPU,cpu))
#$(eval $(call ptmop_pa_tests,debug,DEBUG,debug))
$(eval $(call ptmop_pa_tests,cuda,CUDA,cuda))
#$(eval $(call ptmop_pa_tests,cuda_uvm,CUDA_UVM,cuda:uvm))

$(DATA_DIR):
	ln -s $(SRC)$(DATA_DIR) .

# Testing
MFEM_TESTS = UNIT_TESTS
include $(MFEM_TEST_MK)

ifeq (,$(wildcard $(MFEM_DIR)/../data))
   MFEM_DATA_FLAG =
else
   MFEM_DATA_FLAG = --data $(MFEM_DIR)/../data
endif

%-test-seq: %
	@$(call mfem-test,$<,, Unit tests,$(MFEM_DATA_FLAG),SKIP-NO-VIS)

ceed_tests-test-seq: ceed_tests
	@$(call mfem-test,$<,, CEED Unit tests (cpu),--device ceed-cpu $(MFEM_DATA_FLAG),SKIP-NO-VIS)
ifeq ($(MFEM_USE_CUDA),YES)
	@$(call mfem-test,$<,, CEED Unit tests (cuda-ref),--device ceed-cuda:/gpu/cuda/ref $(MFEM_DATA_FLAG),SKIP-NO-VIS)
	@$(call mfem-test,$<,, CEED Unit tests (cuda-shared),--device ceed-cuda:/gpu/cuda/shared $(MFEM_DATA_FLAG),SKIP-NO-VIS)
	@$(call mfem-test,$<,, CEED Unit tests (cuda-gen),--device ceed-cuda:/gpu/cuda/gen $(MFEM_DATA_FLAG),SKIP-NO-VIS)
endif

RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
%-test-par: %
	@$(call mfem-test,$<, $(RUN_MPI) 1, Parallel unit tests,$(MFEM_DATA_FLAG),SKIP-NO-VIS)
	@$(call mfem-test,$<, $(RUN_MPI) $(MFEM_MPI_NP), Parallel unit tests,$(MFEM_DATA_FLAG),SKIP-NO-VIS)

# Generate an error message if the MFEM library is not built and exit
$(MFEM_LIB_FILE):
	$(error The MFEM library is not built)

clean:
	rm -f $(SEQ_UNIT_TESTS) $(PAR_UNIT_TESTS) *.o */*.o */*~ *~
	rm -rf *.dSYM output_meshes parallel_in_serial.mesh parallel_in_serial.gf