From 18169d3331bbce929c288d5fd6f18c50e854609c Mon Sep 17 00:00:00 2001 From: Dave Roberts Date: Mon, 23 Sep 2024 23:48:43 -0500 Subject: [PATCH 1/2] Add wait-url-change macro to make tests more reliable This macro uses wait-predicate to poll the WebDriver for a change in the URL, thus eliminating some race conditions that were causing flaky test failures. --- CHANGELOG.adoc | 2 + test/etaoin/api_test.clj | 199 ++++++++++++++++++--------------------- 2 files changed, 92 insertions(+), 109 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 5c24dc2..8102496 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -38,6 +38,8 @@ A release with an intentional breaking changes is marked with: ** {issue}656[#656]: Correctly describe behavior when query's parameter is a string. The User Guide and `query` doc strings say that a string passed to `query` is interpreted as an XPath expression. In fact, `query` interprets this as either XPath or CSS depending on the setting of the driver's `:locator` parameter, which can be changed. ({person}dgr[@dgr]) * Quality ** {issue}640[#640]: Significantly improved test coverage. ({person}dgr[@dgr]) +** {issue}646[#646]: Fixed race condition leading to unreliable test. ({person}dgr[@dgr]) +** {issue}674[#674]: Fixed race condition leading to unreliable test. ({person}dgr[@dgr]) == v1.1.41 [minor breaking] - 2024-08-14 [[v1.1.41]] diff --git a/test/etaoin/api_test.clj b/test/etaoin/api_test.clj index 8d3bde1..3a3f5a7 100644 --- a/test/etaoin/api_test.clj +++ b/test/etaoin/api_test.clj @@ -129,6 +129,20 @@ report-browsers test-server) +(defn reload-test-page + [] + (e/go *driver* (test-server-url "test.html")) + (e/wait-visible *driver* {:id :document-end})) + +(defmacro wait-url-change + [& body] + `(let [old-url# (e/get-url *driver*)] + ~@body + (e/wait-predicate (fn [] (not= old-url# (e/get-url *driver*))) + {:timeout 30 ; 30 second timeout total + :interval 0.100 ; + :message "Timeout waiting for URL change"}))) + (deftest test-browser-conditionals (testing "Chrome conditionals" (e/when-chrome *driver* @@ -187,42 +201,28 @@ (is (e/selected? *driver* :vehicle3))) (deftest test-submit - (doto *driver* - (e/fill-multi {:simple-input 1 - :simple-password 2 - :simple-textarea 3}) - (e/submit :simple-input) - ;; Both Safari and Firefox need a slight delay here before the URL - ;; corresponding to the submitted form is valid. Chrome and Edge - ;; seem to do OK without. As of Aug 28, 2024. - (e/when-safari (e/wait 3)) - (e/when-firefox (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=1&password=2&message=3") - is))) + (e/fill-multi *driver* {:simple-input 1 + :simple-password 2 + :simple-textarea 3}) + (wait-url-change + (e/submit *driver* :simple-input)) + (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3"))) (deftest test-input (testing "fill multiple inputs" ;; Test with map form - (doto *driver* - (e/fill-multi {:simple-input 1 - :simple-password 2 - :simple-textarea 3}) - (e/click :simple-submit) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=1&password=2&message=3") - is)) + (e/fill-multi *driver* {:simple-input 1 + :simple-password 2 + :simple-textarea 3}) + (wait-url-change (e/click *driver* :simple-submit)) + (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3")) ;; Test with vector form - (doto *driver* - (e/fill-multi [:simple-input 1 - :simple-password 2 - :simple-textarea 3]) - (e/click :simple-submit) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=1&password=2&message=3") - is))) + (e/fill-multi *driver* + [:simple-input 4 + :simple-password 5 + :simple-textarea 6]) + (wait-url-change (e/click *driver* :simple-submit)) + (is (str/ends-with? (e/get-url *driver*) "?login=4&password=5&message=6"))) (testing "fill-multi bad inputs" (is (thrown+? [:type :etaoin/argument] (e/fill-multi *driver* #{:set :is :not :allowed}))) @@ -231,26 +231,20 @@ (is (thrown+? [:type :etaoin/argument] (e/fill-multi *driver* [:vector :with :odd :length :is :not :allowed])))) (testing "fill human multiple inputs" - (doto *driver* - ;; Test with map form - (e/fill-human-multi {:simple-input "login" - :simple-password "123" - :simple-textarea "text"}) - (e/click :simple-submit) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=login&password=123&message=text") - is)) - (doto *driver* - ;; Test with vector form - (e/fill-human-multi [:simple-input "login" - :simple-password "123" - :simple-textarea "text"]) - (e/click :simple-submit) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=login&password=123&message=text") - is))) + ;; Test with map form + (e/fill-human-multi *driver* + {:simple-input "login" + :simple-password "123" + :simple-textarea "text"}) + (wait-url-change (e/click *driver* :simple-submit)) + (is (str/ends-with? (e/get-url *driver*) "?login=login&password=123&message=text")) + ;; Test with vector form + (e/fill-human-multi *driver* + [:simple-input "login2" + :simple-password "456" + :simple-textarea "text2"]) + (wait-url-change (e/click *driver* :simple-submit)) + (is (str/ends-with? (e/get-url *driver*) "?login=login2&password=456&message=text2"))) (testing "fill-human-multi bad inputs" (is (thrown+? [:type :etaoin/argument] (e/fill-multi *driver* #{:set :is :not :allowed}))) @@ -259,35 +253,27 @@ (is (thrown+? [:type :etaoin/argument] (e/fill-multi *driver* [:vector :with :odd :length :is :not :allowed])))) (testing "fill multiple vars" - (doto *driver* - (e/fill :simple-input 1 "test" 2 \space \A) - (e/click :simple-submit) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=1test2+A&password=&message=") - is))) + (e/fill *driver* :simple-input 1 "test" 2 \space \A) + (wait-url-change (e/click *driver* :simple-submit)) + (is (str/ends-with? (e/get-url *driver*) "?login=1test2+A&password=&message="))) (testing "fill active" - (doto *driver* - (e/click :simple-input) - (e/fill-active "MyLogin") - (e/click :simple-password) - (e/fill-active "MyPassword") - (e/click :simple-textarea) - (e/fill-active "Some text") - (e/click :simple-submit) - (e/when-safari (e/wait 3))) + (e/click *driver* :simple-input) + (e/fill-active *driver* "MyLogin") + (e/click *driver* :simple-password) + (e/fill-active *driver* "MyPassword") + (e/click *driver* :simple-textarea) + (e/fill-active *driver* "Some text") + (wait-url-change (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=MyLogin&password=MyPassword&message=Some+text"))) (testing "fill active human" - (doto *driver* - (e/click :simple-input) - (e/fill-human-active "MyLogin2") - (e/click :simple-password) - (e/fill-human-active "MyPassword2") - (e/click :simple-textarea) - (e/fill-human-active "Some text 2") - (e/click :simple-submit) - (e/when-safari (e/wait 3))) + (e/click *driver* :simple-input) + (e/fill-human-active *driver* "MyLogin2") + (e/click *driver* :simple-password) + (e/fill-human-active *driver* "MyPassword2") + (e/click *driver* :simple-textarea) + (e/fill-human-active *driver* "Some text 2") + (wait-url-change (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=MyLogin2&password=MyPassword2&message=Some+text+2")))) @@ -324,27 +310,27 @@ (deftest test-clear (testing "simple clear" - (doto *driver* - (e/fill {:id :simple-input} "test") - (e/clear {:id :simple-input}) - (e/click {:id :simple-submit}) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=&password=&message=") - is))) + (e/fill *driver* {:id :simple-input} "test") + (e/clear *driver* {:id :simple-input}) + (wait-url-change (e/click *driver* {:id :simple-submit})) + (is (str/ends-with? (e/get-url *driver*) "?login=&password=&message="))) (testing "multiple clear" - (doto *driver* - (e/fill-multi {:simple-input 1 - :simple-password 2 - :simple-textarea 3}) - (e/clear :simple-input - :simple-password - :simple-textarea) - (e/when-safari (e/wait 3)) - (-> e/get-url - (str/ends-with? "?login=&password=&message=") - is)))) + ;; Note that we need to reload the test page here because the URL + ;; from the first test is the same as the second and thus if we + ;; didn't do this we couldn't detect whether anything happened + ;; after the first test. + (reload-test-page) + (e/fill-multi *driver* + {:simple-input 1 + :simple-password 2 + :simple-textarea 3}) + (e/clear *driver* + :simple-input + :simple-password + :simple-textarea) + (wait-url-change (e/click *driver* {:id :simple-submit})) + (is (str/ends-with? (e/get-url *driver*) "?login=&password=&message=")))) (deftest test-enabled (doto *driver* @@ -427,14 +413,10 @@ (is (= inner-html result)))) (deftest test-title - (doto *driver* - (-> e/get-title (= "Webdriver Test Document") is))) + (is (= (e/get-title *driver*) "Webdriver Test Document"))) (deftest test-url - (doto *driver* - (-> e/get-url - (str/ends-with? "/test.html") - is))) + (is (str/ends-with? (e/get-url *driver*) "/test.html"))) (deftest test-css-props (testing "single css" @@ -821,12 +803,11 @@ (deftest test-set-hash (testing "set hash" - (doto *driver* - (e/set-hash "hello") - (-> e/get-hash (= "hello") is) - (-> e/get-url (str/ends-with? "/test.html#hello") is) - (e/set-hash "goodbye") - (-> e/get-url (str/ends-with? "/test.html#goodbye") is)))) + (e/set-hash *driver* "hello") + (is (= (e/get-hash *driver*) "hello")) + (is (str/ends-with? (e/get-url *driver*) "/test.html#hello")) + (e/set-hash *driver* "goodbye") + (is (str/ends-with? (e/get-url *driver*) "/test.html#goodbye")))) (deftest test-query (testing "finding an element by id keyword" @@ -1159,8 +1140,8 @@ (e/add-pointer-click-el textarea) e/add-pause (e/add-pointer-click-el submit))] - (e/perform-actions *driver* keyboard mouse) - (e/wait 1) + (wait-url-change + (e/perform-actions *driver* keyboard mouse)) (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3"))))) (deftest test-shadow-dom From 3b1e29c6ec09803cdb7d06de95e04f8a341747b3 Mon Sep 17 00:00:00 2001 From: Dave Roberts Date: Tue, 24 Sep 2024 09:48:26 -0500 Subject: [PATCH 2/2] Update wait-url-change to wait for URL to match a regex In addition to waiting for the URL to change, wait-url-change also waits for the new URL to match a regex (with re-find). --- test/etaoin/api_test.clj | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/etaoin/api_test.clj b/test/etaoin/api_test.clj index 3a3f5a7..f95fd4e 100644 --- a/test/etaoin/api_test.clj +++ b/test/etaoin/api_test.clj @@ -135,12 +135,14 @@ (e/wait-visible *driver* {:id :document-end})) (defmacro wait-url-change - [& body] + [re & body] `(let [old-url# (e/get-url *driver*)] ~@body - (e/wait-predicate (fn [] (not= old-url# (e/get-url *driver*))) + (e/wait-predicate (fn [] (let [new-url# (e/get-url *driver*)] + (and (not= old-url# new-url#) + (re-find ~re new-url#)))) {:timeout 30 ; 30 second timeout total - :interval 0.100 ; + :interval 0.100 ; poll at 100 msec interval :message "Timeout waiting for URL change"}))) (deftest test-browser-conditionals @@ -204,7 +206,7 @@ (e/fill-multi *driver* {:simple-input 1 :simple-password 2 :simple-textarea 3}) - (wait-url-change + (wait-url-change #"login" (e/submit *driver* :simple-input)) (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3"))) @@ -214,14 +216,14 @@ (e/fill-multi *driver* {:simple-input 1 :simple-password 2 :simple-textarea 3}) - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3")) ;; Test with vector form (e/fill-multi *driver* [:simple-input 4 :simple-password 5 :simple-textarea 6]) - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=4&password=5&message=6"))) (testing "fill-multi bad inputs" (is (thrown+? [:type :etaoin/argument] @@ -236,14 +238,14 @@ {:simple-input "login" :simple-password "123" :simple-textarea "text"}) - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=login&password=123&message=text")) ;; Test with vector form (e/fill-human-multi *driver* [:simple-input "login2" :simple-password "456" :simple-textarea "text2"]) - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=login2&password=456&message=text2"))) (testing "fill-human-multi bad inputs" (is (thrown+? [:type :etaoin/argument] @@ -254,7 +256,7 @@ (e/fill-multi *driver* [:vector :with :odd :length :is :not :allowed])))) (testing "fill multiple vars" (e/fill *driver* :simple-input 1 "test" 2 \space \A) - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=1test2+A&password=&message="))) (testing "fill active" (e/click *driver* :simple-input) @@ -263,7 +265,7 @@ (e/fill-active *driver* "MyPassword") (e/click *driver* :simple-textarea) (e/fill-active *driver* "Some text") - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=MyLogin&password=MyPassword&message=Some+text"))) (testing "fill active human" @@ -273,7 +275,7 @@ (e/fill-human-active *driver* "MyPassword2") (e/click *driver* :simple-textarea) (e/fill-human-active *driver* "Some text 2") - (wait-url-change (e/click *driver* :simple-submit)) + (wait-url-change #"login" (e/click *driver* :simple-submit)) (is (str/ends-with? (e/get-url *driver*) "?login=MyLogin2&password=MyPassword2&message=Some+text+2")))) @@ -312,7 +314,7 @@ (testing "simple clear" (e/fill *driver* {:id :simple-input} "test") (e/clear *driver* {:id :simple-input}) - (wait-url-change (e/click *driver* {:id :simple-submit})) + (wait-url-change #"login" (e/click *driver* {:id :simple-submit})) (is (str/ends-with? (e/get-url *driver*) "?login=&password=&message="))) (testing "multiple clear" @@ -329,7 +331,7 @@ :simple-input :simple-password :simple-textarea) - (wait-url-change (e/click *driver* {:id :simple-submit})) + (wait-url-change #"login" (e/click *driver* {:id :simple-submit})) (is (str/ends-with? (e/get-url *driver*) "?login=&password=&message=")))) (deftest test-enabled @@ -1140,7 +1142,7 @@ (e/add-pointer-click-el textarea) e/add-pause (e/add-pointer-click-el submit))] - (wait-url-change + (wait-url-change #"login" (e/perform-actions *driver* keyboard mouse)) (is (str/ends-with? (e/get-url *driver*) "?login=1&password=2&message=3")))))