#!/usr/bin/env python3 # Copyright 2023 The IREE Authors # # Licensed under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception import pathlib import unittest import configure_ci SORTED_DEFAULT_BENCHMARK_PRESETS_STR = ",".join( sorted(configure_ci.DEFAULT_BENCHMARK_PRESET_GROUP) ) class ConfigureCITest(unittest.TestCase): def test_get_benchmark_presets_no_preset(self): presets_str = configure_ci.get_benchmark_presets( trailers={}, labels=["unrelated-labels"], is_pr=True, is_llvm_integrate_pr=False, ) self.assertEqual(presets_str, "") def test_get_benchmark_presets_from_pr_labels(self): presets_str = configure_ci.get_benchmark_presets( trailers={}, labels=["benchmarks:x86_64", "benchmarks:cuda"], is_pr=True, is_llvm_integrate_pr=False, ) self.assertEqual(presets_str, "comp-stats,cuda,x86_64") def test_get_benchmark_presets_from_trailers_and_labels(self): presets_str = configure_ci.get_benchmark_presets( trailers={"benchmark-extra": "android-cpu,cuda-large,x86_64-large"}, labels=["benchmarks:vulkan-nvidia"], is_pr=True, is_llvm_integrate_pr=False, ) self.assertEqual( presets_str, "android-cpu,comp-stats,cuda-large,vulkan-nvidia,x86_64-large" ) def test_get_benchmark_presets_from_default_group(self): presets_str = configure_ci.get_benchmark_presets( trailers={"benchmark-extra": "default"}, labels=[], is_pr=True, is_llvm_integrate_pr=False, ) self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) # Sanity check to ensure no `*-large` preset in the default group. self.assertNotIn("-large", presets_str) def test_get_benchmark_presets_for_non_pr(self): presets_str = configure_ci.get_benchmark_presets( trailers={}, labels=[], is_pr=False, is_llvm_integrate_pr=False ) self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) def test_get_benchmark_presets_for_llvm_integrate_pr(self): presets_str = configure_ci.get_benchmark_presets( trailers={}, labels=[], is_pr=True, is_llvm_integrate_pr=True ) self.assertEqual(presets_str, SORTED_DEFAULT_BENCHMARK_PRESETS_STR) # Sample PR description: # ``` # PR Title # # PR body... # # skip-llvm-integrate-benchmark: some good reasons # ``` # Result: No benchmark is automatically enabled on the LLVM integrate PR. def test_get_benchmark_presets_skip_llvm_integrate_benchmark(self): presets_str = configure_ci.get_benchmark_presets( trailers={"skip-llvm-integrate-benchmark": "some good reasons"}, labels=[], is_pr=True, is_llvm_integrate_pr=True, ) self.assertEqual(presets_str, "") def test_get_benchmark_presets_unknown_preset(self): self.assertRaises( ValueError, lambda: configure_ci.get_benchmark_presets( trailers={"benchmark-extra": "unknown"}, labels=[], is_pr=True, is_llvm_integrate_pr=False, ), ) def test_parse_jobs_trailer(self): trailers = {"key": "job1,job2"} key = "key" all_jobs = {"job1", "job2", "job3"} jobs = configure_ci.parse_jobs_trailer(trailers, key, all_jobs) self.assertCountEqual(jobs, {"job1", "job2"}) def test_parse_jobs_trailer_whitespace(self): trailers = {"key": " job1 , job2 "} key = "key" all_jobs = {"job1", "job2", "job3"} jobs = configure_ci.parse_jobs_trailer(trailers, key, all_jobs) self.assertCountEqual(jobs, {"job1", "job2"}) def test_parse_jobs_trailer_all_with_others(self): bad_text = "job1, all" trailers = {"key": bad_text} key = "key" all_jobs = {"job1", "job2", "job3"} with self.assertRaises(ValueError) as cm: configure_ci.parse_jobs_trailer(trailers, key, all_jobs) msg = str(cm.exception) self.assertIn(configure_ci.ALL_KEY, msg) self.assertIn(bad_text, msg) def test_parse_jobs_unknown_job(self): unknown_job = "unknown_job" trailers = {"key": f"job1, {unknown_job}"} key = "key" all_jobs = {"job1", "job2", "job3"} # Unknown jobs log a warning, as multiple workflows use configure_ci # and a name may be recognized by one workflow and not another. jobs = configure_ci.parse_jobs_trailer(trailers, key, all_jobs) self.assertCountEqual(jobs, {"job1"}) def test_get_enabled_jobs_all(self): trailers = {} all_jobs = {"job1", "job2", "job3"} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, all_jobs) def test_get_enabled_jobs_postsubmit(self): trailers = {} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = False is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, all_jobs) def test_get_enabled_jobs_no_postsubmit(self): trailers = {} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, default_jobs) def test_get_enabled_jobs_llvm_integrate(self): trailers = {} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = True modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, all_jobs) def test_get_enabled_jobs_no_modifies(self): trailers = {} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["experimental/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, {}) def test_get_enabled_jobs_skip(self): trailers = {configure_ci.Trailer.SKIP_JOBS: "job1,job2"} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, {"job3"}) def test_get_enabled_jobs_skip_all(self): trailers = {configure_ci.Trailer.SKIP_JOBS: "all"} default_jobs = {"job1", "job2", "job3"} postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, {}) def test_get_enabled_jobs_extra(self): postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) trailers = {configure_ci.Trailer.EXTRA_JOBS: postsubmit_job} default_jobs = {"job1", "job2", "job3"} all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, all_jobs) def test_get_enabled_jobs_exactly(self): postsubmit_job = next(iter(configure_ci.DEFAULT_POSTSUBMIT_ONLY_JOBS)) trailers = {configure_ci.Trailer.EXACTLY_JOBS: postsubmit_job} default_jobs = {"job1", "job2", "job3"} all_jobs = default_jobs | {postsubmit_job} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) self.assertCountEqual(jobs, {postsubmit_job}) def test_get_enabled_jobs_metal(self): trailers = {} all_jobs = {"job1"} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/src/iree/hal/drivers/metal/file"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) expected_jobs = {"job1", "build_test_all_macos_arm64"} self.assertCountEqual(jobs, expected_jobs) def test_get_enabled_jobs_windows(self): trailers = {} all_jobs = {"job1"} is_pr = True is_llvm_integrate_pr = False modified_paths = ["runtime/src/iree/base/internal/threading_win32.c"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) expected_jobs = {"job1", "build_test_all_windows"} self.assertCountEqual(jobs, expected_jobs) def test_get_enabled_jobs_windows_docs(self): # docs/ directory is excluded from CI, superceding "windows" inclusion trailers = {} all_jobs = {"job1"} is_pr = True is_llvm_integrate_pr = False modified_paths = ["docs/windows.md"] jobs = configure_ci.get_enabled_jobs( trailers, all_jobs, modified_paths=modified_paths, is_pr=is_pr, is_llvm_integrate_pr=is_llvm_integrate_pr, ) expected_jobs = {} self.assertCountEqual(jobs, expected_jobs) def test_parse_path_from_workflow_ref(self): path = configure_ci.parse_path_from_workflow_ref( "octocat/example", "octocat/example/.github/test.yml@1234" ) self.assertEqual(path, pathlib.Path(".github/test.yml")) def test_parse_path_from_workflow_ref_invalid_ref(self): self.assertRaises( ValueError, lambda: configure_ci.parse_path_from_workflow_ref( "octocat/example", "squid/unknown/.github/test.yml@1234" ), ) if __name__ == "__main__": unittest.main()