[Tests] Fix tests 4th ed. (#220)

1. Record tests conditions in log file.
2. Fix op name.
3. Print mode in benchmark results.
4. Check pytest.mark.{OP_NAME} in CI.

---------

Co-authored-by: zhengyang <[email protected]>

Author: zhzhcookie

OperatorList.md

@@ -29,29 +29,28 @@
 - mean
 
 ## v2.0
-
-- mv
 - all
 - any
 - bitwise_and
 - bitwise_not
 - bitwise_or
 - cos
+- clamp
 - eq
 - ge
 - gt
+- mv
 - isinf
 - isnan
 - le
 - lt
 - ne
 - neg
-- or
+- or_
 - sin
 - tanh
 - amax
 - argmax
-- clamp
 - max
 - min
 - outer
@@ -66,13 +65,12 @@
 - sigmoid
 
 ## v3.0
-
 - _conv_depthwise2d
 - _convolution
 - conv1d
 - conv2d
 - convolution
-- cudnn_convolution
+- cudnn_convolution (N/A)
 - multinomial
 - nonzero
 - normal
@@ -99,12 +97,11 @@
 - resolve_neg
 - arange
 - cat
-- chunk
-- chunk
+- chunk (N/A)
 - concat
 - constant_pad_nd
-- contiguous
-- copy_
+- contiguous (N/A)
+- copy_ (N/A)
 - fill
 - flip
 - full
@@ -117,30 +114,48 @@
 - narrow
 - ones
 - pad
-- permute
+- permute (N/A)
 - repeat
 - repeat_interleave
-- resize
+- resize_ (N/A)
 - scatter
-- select
+- index_add
+- select (N/A)
 - select_scatter
 - slice
 - slice_scatter
 - sort
-- split
-- split_with_sizes
+- split (N/A)
+- split_with_sizes (N/A)
 - stack
 - tile
-- transpose
-- unfold
+- transpose (N/A)
+- unfold (N/A)
 - where
 - zeros
 
 ## v4.0
-
 - rms_norm
-- skip_layernorm
-- skip_rmsnorm
-- apply_rotary_position_embedding
-- silu_and_mul
+- skip_rms_norm
+- skip_layer_norm
 - gelu_and_mul
+- silu_and_mul
+- apply_rotary_pos_emb
+
+## v5.0
+- unique
+- isin
+- allclose
+- isclose
+- isfinite
+- exponential_
+- vstack
+- minimum
+- maximum
+- ones_like
+- zeros_like
+- randn_like
+- floor_divide
+- masked_select
+- trunc_divide
+- remainder

benchmark/performance_utils.py

@@ -66,8 +66,12 @@ def profile(self, op, *args, **kwargs):
         return latency
 
     def run(self):
+        mode_str = "cpu" if CPU_MODE else "cuda"
+        print("")
         for dtype in self.dtypes:
-            print(f"Operator {self.op_name} Performance Test ({dtype})")
+            print(
+                f"Operator {self.op_name} Performance Test (dtype={dtype}, mode={mode_str})"
+            )
             print("Size    Torch Latency (ms)    Gems Latency (ms)    Gems Speedup")
             print("---------------------------------------------------------------")
             for size in self.sizes:

tests/conftest.py

@@ -1,3 +1,7 @@
+import json
+import logging
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--ref",
@@ -15,6 +19,14 @@ def pytest_addoption(parser):
         choices=["normal", "quick"],
         help="run tests on normal or quick mode",
     )
+    parser.addoption(
+        "--record",
+        action="store",
+        default="none",
+        required=False,
+        choices=["none", "log"],
+        help="tests function param recorded in log files or not",
+    )
 
 
 def pytest_configure(config):
@@ -23,3 +35,60 @@ def pytest_configure(config):
 
     global QUICK_MODE
     QUICK_MODE = config.getoption("--mode") == "quick"
+
+    global RECORD_LOG
+    RECORD_LOG = config.getoption("--record") == "log"
+    if RECORD_LOG:
+        global RUNTEST_INFO, BUILTIN_MARKS, REGISTERED_MARKERS
+        RUNTEST_INFO = {}
+        BUILTIN_MARKS = {
+            "parametrize",
+            "skip",
+            "skipif",
+            "xfail",
+            "usefixtures",
+            "filterwarnings",
+            "timeout",
+            "tryfirst",
+            "trylast",
+        }
+        REGISTERED_MARKERS = {
+            marker.split(":")[0].strip() for marker in config.getini("markers")
+        }
+        cmd_args = [
+            arg.replace(".py", "").replace("=", "_").replace("/", "_")
+            for arg in config.invocation_params.args
+        ]
+        logging.basicConfig(
+            filename="result_{}.log".format("_".join(cmd_args)).replace("_-", "-"),
+            filemode="w",
+            level=logging.INFO,
+            format="[%(levelname)s] %(message)s",
+        )
+
+
+def pytest_runtest_teardown(item, nextitem):
+    if not RECORD_LOG:
+        return
+    if hasattr(item, "callspec"):
+        all_marks = list(item.iter_markers())
+        op_marks = [
+            mark.name
+            for mark in all_marks
+            if mark.name not in BUILTIN_MARKS and mark.name not in REGISTERED_MARKERS
+        ]
+        if len(op_marks) > 0:
+            params = str(item.callspec.params)
+            for op_mark in op_marks:
+                if op_mark not in RUNTEST_INFO:
+                    RUNTEST_INFO[op_mark] = [params]
+                else:
+                    RUNTEST_INFO[op_mark].append(params)
+        else:
+            func_name = item.function.__name__
+            logging.warning("There is no mark at {}".format(func_name))
+
+
+def pytest_sessionfinish(session, exitstatus):
+    if RECORD_LOG:
+        logging.info(json.dumps(RUNTEST_INFO, indent=2))

tests/test_binary_pointwise_ops.py

@@ -142,6 +142,7 @@ def test_accuracy_bitwiseand_scalar_tensor(shape, dtype):
     gems_assert_equal(res_out, ref_out)
 
 
+@pytest.mark.or_
 @pytest.mark.bitwise_or
 @pytest.mark.parametrize("shape", POINTWISE_SHAPES)
 @pytest.mark.parametrize("dtype", INT_DTYPES + BOOL_TYPES)
@@ -166,6 +167,7 @@ def test_accuracy_bitwiseor(shape, dtype):
     gems_assert_equal(res_out, ref_out)
 
 
+@pytest.mark.or_
 @pytest.mark.bitwise_or
 @pytest.mark.parametrize("shape", POINTWISE_SHAPES)
 @pytest.mark.parametrize("dtype", INT_DTYPES + BOOL_TYPES)
@@ -187,6 +189,7 @@ def test_accuracy_bitwiseor_scalar(shape, dtype):
     gems_assert_equal(res_out, ref_out)
 
 
+@pytest.mark.or_
 @pytest.mark.bitwise_or
 @pytest.mark.parametrize("shape", POINTWISE_SHAPES)
 @pytest.mark.parametrize("dtype", INT_DTYPES + BOOL_TYPES)

tests/test_special_ops.py

@@ -668,6 +668,7 @@ def test_accuracy_cat(shape, dim, dtype):
 ]
 
 
+@pytest.mark.vstack
 @pytest.mark.parametrize("shape", VSTACK_SHAPES)
 @pytest.mark.parametrize("dtype", FLOAT_DTYPES + INT_DTYPES)
 def test_accuracy_vstack(shape, dtype):

tools/op-unit-test.sh

@@ -18,6 +18,7 @@ PR_ID_DIR="PR${PR_ID}"
 
 COVERAGE_ARGS="--parallel-mode --omit "*/.flaggems/*","*/usr/lib/*" --source=./src,./tests --data-file=${ID_SHA}-op"
 cmds=(
+   "bash tools/pytest_mark_check.sh &"
    "CUDA_VISIBLE_DEVICES=0 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_blas_ops.py &"
    "CUDA_VISIBLE_DEVICES=1 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_reduction_ops.py &"
    "CUDA_VISIBLE_DEVICES=2 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_general_reduction_ops.py &"
@@ -31,7 +32,8 @@ cmds=(
     CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_libentry.py && \
     CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_pointwise_dynamic.py && \
     CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_shape_utils.py && \
-    CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_tensor_wrapper.py &"
+    CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_tensor_wrapper.py && \
+    CUDA_VISIBLE_DEVICES=6 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_unary_pointwise_ops.py -m abs --record=log &"
    "CUDA_VISIBLE_DEVICES=7 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_blas_ops.py --ref=cpu --mode=quick && \
     CUDA_VISIBLE_DEVICES=7 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_reduction_ops.py --ref=cpu --mode=quick && \
     CUDA_VISIBLE_DEVICES=7 coverage run ${COVERAGE_ARGS} -m pytest -s tests/test_general_reduction_ops.py --ref=cpu --mode=quick && \

tools/pytest_mark_check.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$BASH_VERSION" ]; then
+    echo "[ERROR]This script must be run using bash!" >&2
+    exit 1
+fi
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+TEST_FILES="${SCRIPT_DIR}/../tests/test_*.py"
+MD_FILE="${SCRIPT_DIR}/../OperatorList.md"
+
+# all
+grep 'pytest.mark.' ${TEST_FILES} |grep -v parametrize |grep -v 'skip(' |grep -v 'skipif(' |awk -F':@pytest.mark.' '{print $1"\t"$2}' |sort -k 2 |uniq >mark.txt
+
+# unique
+grep 'pytest.mark.' ${TEST_FILES} |grep -v parametrize |grep -v 'skip(' |grep -v 'skipif(' |awk -F':@pytest.mark.' '{print $2}' |sort |uniq |awk '!seen[$1]++' >mark.uniq.txt
+
+# recorded in MD_FILE
+grep '-' "${MD_FILE}" |grep -v '#' |grep -v 'N/A' |awk '{print $2}' |sort >mark.md.txt
+
+MARK_NUM=`wc -l mark.txt |awk '{print $1}'`
+MARK_UNIQ_NUM=`wc -l mark.uniq.txt |awk '{print $1}'`
+MARK_MD_NUM=`wc -l mark.md.txt |awk '{print $1}'`
+echo "Generated successfully: mark.txt (${MARK_NUM}), mark.uniq.txt (${MARK_UNIQ_NUM}), mark.md.txt (${MARK_MD_NUM})"
+
+echo "-------- diff mark.uniq.txt mark.md.txt --------"
+diff mark.uniq.txt mark.md.txt || true
+echo "------------------------------------------------"
+
+TEST_OP_FILES=`ls ${TEST_FILES} |grep "_ops.py" |grep -v "test_named_ops.py"`
+EXCLUDED_MARKS=("pytest.mark.parametrize\(" \
+                "pytest.mark.skip\(" \
+                "pytest.mark.skipif\(" \
+                "pytest.mark.xfail\(" \
+                "pytest.mark.usefixtures\(" \
+                "pytest.mark.filterwarnings\(" \
+                "pytest.mark.timeout\(" \
+                "pytest.mark.tryfirst\(" \
+                "pytest.mark.trylast\(")
+
+test_file_count=0
+for file in ${TEST_OP_FILES}; do
+    echo "Checking file: ${file}"
+    set +e
+
+    awk -v marks="${EXCLUDED_MARKS[*]}" '
+    BEGIN {
+        test_func = 0; decorated = 0; error = 0;
+        split(marks, excluded_marks, " ")
+    }
+
+    /^@pytest\.mark\./ {
+        test_func = 1
+        excluded = 0
+        for (i in excluded_marks) {
+            if ($0 ~ excluded_marks[i]) {
+                excluded = 1
+                break
+            }
+        }
+        if (excluded == 0) {
+            decorated = 1
+        }
+        next
+    }
+
+    /^def / {
+        if (test_func == 1) {
+            if (decorated == 0) {
+                print "[ERROR]"$0
+                error = 1
+            }
+            test_func = 0
+            decorated = 0
+        }
+    }
+
+    END {
+        if (error == 1) {
+            exit 1
+        }
+    }
+    ' "$file"
+
+    if [ $? -ne 0 ]; then
+        echo "[ERROR]There are some test_op_func without 'pytest.mark.{OP_NAME}' in ${file}"
+        exit 1
+    fi
+
+    set -e
+    test_file_count=$((test_file_count + 1))
+done
+
+echo "Finish checking ${test_file_count} files successfully."