From 307087f6b5acf173f72ff8d8b8871a73b96605b7 Mon Sep 17 00:00:00 2001
From: Shunsuke Shibayama <sbym1346@gmail.com>
Date: Tue, 14 Feb 2023 11:30:23 +0900
Subject: [PATCH] test: let some tests not to run locally

---
 .github/workflows/rust.yml      |  4 ++--
 doc/EN/dev_guide/test.md        |  9 +++++++++
 doc/JA/dev_guide/test.md        |  9 +++++++++
 tests/eval/build_in_function.rs | 18 ++++++++++++++++++
 tests/eval/literal.rs           | 18 ++++++++++++++++++
 tests/repl.rs                   |  1 +
 6 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 278203de6..20560a22e 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -46,10 +46,10 @@ jobs:
       - name: Run tests (Windows)
         if: runner.os == 'Windows'
         # HACK: The cause is unknown, but on windows, the exit code is 1 even if tests are successful.
-        run: cargo test --features large_thread --features pre-commit --all --verbose
+        run: cargo test --features large_thread --features pre-commit --all --verbose -- --include-ignored
       - name: Run tests (Other OS)
         if: runner.os != 'Windows'
-        run: cargo test --features large_thread --all --verbose
+        run: cargo test --features large_thread --all --verbose -- --include-ignored
 
   build-check:
     strategy:
diff --git a/doc/EN/dev_guide/test.md b/doc/EN/dev_guide/test.md
index 48dc28dad..769f26852 100644
--- a/doc/EN/dev_guide/test.md
+++ b/doc/EN/dev_guide/test.md
@@ -21,3 +21,12 @@ A positive test is a test to check whether the compiler operates as intended, an
 Due to the nature of programming language processors, among all software, they are particularly susceptible to invalid input, and errors must always be presented to the user, so the latter must also be taken care of.
 
 If you add a new feature to the language, you need to write at least one positive test. Also, please write negative tests if possible.
+
+## `#[ignore]` attribute
+
+The Erg development team recommends pre-commit.
+This prevents bugs from getting into the code by running tests before each commit, but some tests are time-consuming and slow down the commit.
+
+Therefore, tests that are heavy or have a low probability of failure are marked with the `#[ignore]` attribute.
+Tests with the `#[ignore]` attribute are not run by `cargo test`, but can be run with `cargo test -- --include-ignored`.
+These tests are run by CI and do not need to be run on the local PC.
diff --git a/doc/JA/dev_guide/test.md b/doc/JA/dev_guide/test.md
index 1c25287ba..94045f105 100644
--- a/doc/JA/dev_guide/test.md
+++ b/doc/JA/dev_guide/test.md
@@ -21,3 +21,12 @@ cargoはテスト実行用のスレッドを小さく取るため、`large_threa
 プログラミング言語処理系はその性質上、あらゆるソフトウェアの中でも特に不正な入力を受けやすく、かつエラーを常にユーザーに提示しなくてはならないので、後者にも気を配る必要があります。
 
 あなたが言語に新しい機能を追加するならば、少なくとも1つの正常系テストを書く必要があります。異常系テストも出来れば書いてください。
+
+## ignore属性
+
+Erg開発チームはpre-commitの導入を推奨しています。
+これにより、コミット前に毎回テストが走るためバグの混入を未然に防ぐことができますが、幾つかのテストは時間がかかるためコミットが遅くなってしまいます。
+
+そこで、重いテストないし失敗する蓋然性が低いテストには`#[ignore]`属性を付けています。
+`#[ignore]`属性を付けたテストは`cargo test`では実行されませんが、`cargo test -- --include-ignored`で実行することができます。
+これらのテストはCIで実行されるため、ローカルPCで実行する必要はありません。
diff --git a/tests/eval/build_in_function.rs b/tests/eval/build_in_function.rs
index 6eaa400cc..8c6085f6a 100644
--- a/tests/eval/build_in_function.rs
+++ b/tests/eval/build_in_function.rs
@@ -3,36 +3,43 @@
 use crate::eval::{eval, successful_output};
 
 #[test]
+#[ignore]
 fn eval_print_1() {
     assert_eq!(eval("print! 1"), successful_output("1\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_print_str_1() {
     assert_eq!(eval("print! \"abc\""), successful_output("abc\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_print_str_2() {
     assert_eq!(eval("print!(\"a\")"), successful_output("a\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_print_ratio() {
     assert_eq!(eval("print! \"0.3\""), successful_output("0.3\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_print_bool() {
     assert_eq!(eval("print! True"), successful_output("True\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_print_unit() {
     assert_eq!(eval("print! (())"), successful_output("()\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_interpolation_1() {
     assert_eq!(
         eval("world = \"world\"\nprint! \"hello \\{world}\""),
@@ -41,11 +48,13 @@ fn eval_interpolation_1() {
 }
 
 #[test]
+#[ignore]
 fn eval_interpolation_2() {
     assert_eq!(eval("print! \"\\{0.005}\""), successful_output("0.005\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_multiline_str() {
     assert_eq!(
         eval(
@@ -58,6 +67,7 @@ D""""#
 }
 
 #[test]
+#[ignore]
 fn eval_keyword_call() {
     assert_eq!(
         eval("print! \"a\", \"b\", 3, end := \"\""),
@@ -66,6 +76,7 @@ fn eval_keyword_call() {
 }
 
 #[test]
+#[ignore]
 fn eval_invalid_print() {
     let output = eval("print 1"); // print! is correct
     assert_eq!(output.stdout, "");
@@ -74,26 +85,31 @@ fn eval_invalid_print() {
 }
 
 #[test]
+#[ignore]
 fn eval_assign_and_print() {
     assert_eq!(eval("num = -3\nprint! num * 2").stdout, "-6\n");
 }
 
 #[test]
+#[ignore]
 fn eval_assert_true() {
     assert_eq!(eval("assert True"), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_1() {
     assert_eq!(eval("assert 1"), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_assign_and_assert() {
     assert_eq!(eval("flag = True\nassert flag"), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_false() {
     let output = eval("assert False");
     assert_eq!(output.stdout, "");
@@ -102,11 +118,13 @@ fn eval_assert_false() {
 }
 
 #[test]
+#[ignore]
 fn eval_assert_0point2() {
     assert_eq!(eval("assert 0.2").status_code, Some(1));
 }
 
 #[test]
+#[ignore]
 fn eval_invalid_assert() {
     assert_eq!(eval("assert! True").status_code, Some(1));
 }
diff --git a/tests/eval/literal.rs b/tests/eval/literal.rs
index 3e4028163..dcbde94ad 100644
--- a/tests/eval/literal.rs
+++ b/tests/eval/literal.rs
@@ -1,6 +1,7 @@
 use crate::eval::{eval, successful_output};
 
 #[test]
+#[ignore]
 fn eval_assert_str() {
     assert_eq!(
         eval("assert \"abcdef\" == \"abcdef\""),
@@ -9,6 +10,7 @@ fn eval_assert_str() {
 }
 
 #[test]
+#[ignore]
 fn eval_assert_interpolation() {
     assert_eq!(
         eval("assert \"1234567890ABC\" == \"\\{1234567890}ABC\""),
@@ -17,16 +19,19 @@ fn eval_assert_interpolation() {
 }
 
 #[test]
+#[ignore]
 fn eval_print_empty() {
     assert_eq!(eval("print! \"\""), successful_output("\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_empty() {
     assert_eq!(eval("assert \"\" == \"\""), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_interpolation_2() {
     assert_eq!(
         eval(r#"a = 10;assert "\{2 * 5}" == "\{a}""#),
@@ -35,6 +40,7 @@ fn eval_assert_interpolation_2() {
 }
 
 #[test]
+#[ignore]
 fn eval_interpolation() {
     assert_eq!(
         eval(r#"print! "    \{"b"}\{False} \{[1]}""#),
@@ -43,6 +49,7 @@ fn eval_interpolation() {
 }
 
 #[test]
+#[ignore]
 fn eval_interpolation_2() {
     assert_eq!(
         eval(r#"print! "a\{"b"}c\{"d \{" e\{"f"}g\{-2+3}"}"}""#),
@@ -51,6 +58,7 @@ fn eval_interpolation_2() {
 }
 
 #[test]
+#[ignore]
 fn eval_multiline_string() {
     assert_eq!(
         eval(
@@ -64,6 +72,7 @@ j kl """"#
 }
 
 #[test]
+#[ignore]
 fn eval_multiline_string_interpolation() {
     assert_eq!(
         eval(
@@ -78,6 +87,7 @@ a
 }
 
 #[test]
+#[ignore]
 fn eval_invalid_assertion() {
     let output = eval("assert \"abcde\" == \"abcdef\"");
     assert_eq!(output.stdout, "");
@@ -86,26 +96,31 @@ fn eval_invalid_assertion() {
 }
 
 #[test]
+#[ignore]
 fn eval_invalid_closing_string() {
     assert_eq!(eval("print! \"\\\"").status_code, Some(1));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_99() {
     assert_eq!(eval("assert 99 == 99"), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_assert_minus2() {
     assert_eq!(eval("assert -2 == -2"), successful_output(""));
 }
 
 #[test]
+#[ignore]
 fn eval_minus1000() {
     assert_eq!(eval("print! -1000"), successful_output("-1000\n"));
 }
 
 #[test]
+#[ignore]
 fn eval_0_eq_0() {
     assert_eq!(eval("print! 0 == 0"), successful_output("True\n"));
 }
@@ -124,6 +139,7 @@ fn eval_neg_bignum() {
 */
 
 #[test]
+#[ignore]
 fn eval_assert_inequality() {
     let result = eval("assert 100 == 1000");
     assert_eq!(result.stdout, "");
@@ -132,11 +148,13 @@ fn eval_assert_inequality() {
 }
 
 #[test]
+#[ignore]
 fn eval_assert_inequality_2() {
     assert_eq!(eval("assert 10 == 11").status_code, Some(1));
 }
 
 #[test]
+#[ignore]
 fn eval_ratio() {
     assert_eq!(eval("print! 0.1234"), successful_output("0.1234\n"));
 }
diff --git a/tests/repl.rs b/tests/repl.rs
index 1a2f64568..b09772ad0 100644
--- a/tests/repl.rs
+++ b/tests/repl.rs
@@ -36,6 +36,7 @@ fn exec_repl_for_loop() -> Result<(), ()> {
 }
 
 #[test]
+#[ignore]
 fn exec_repl_auto_indent_dedent_check() -> Result<(), ()> {
     expect_repl_success(
         "repl_auto_indent_dedent",