Updating makefile commands (#1226)

* add single schema validation and the option to run a single functional test

* Allow lint and tests to be run in one command for Python and JS

* Allow functional tests to be run headless

* allow dev folders such as venv and contents of gitignore to be ignored when calling make format and make lint

* allow a chosen suite to be run

* Exclude venv and tests directory to ensure python lint runs without errors

* add a command that runs JS lint and test headless and made commands names consistent

* Rename the commands to clarify the order of linting and testing when calling the command on JS/Python

* Add extra commands README that developers can utilise

* Remove venv from being ignored

* Refactor commands to clarify the purpose and to ensure consistency in names

* Alter commands that developers find useful

* Improve the description of  commands used for functional tests

* Fix config and ignore files

* Replace yarn commands with make commands

* Update the file to make use of the new Make commands

* Add new commands to generate pages and to generate new specs given a schema name

* Replace yarn commands with Make commands

* Refactor issues with running make commands

* Remove incorrect pipenv commands

* Clarify which suites are available to run using the test-functional-suite command

* Correct the make commands to ensure the PR tests pass correctly

* Update commands to use yarn commands directly rather than shell scripts

* Remove --suite from generate_pages command

* Temporary error to test if pullrequest.yml works as intended

* Correct deliberate error to ensure pullrequest.yml works as intended

* Delete unused file

* Remove the .json to be required when validating a single schema

.github/workflows/pull_request.yml

@@ -73,7 +73,7 @@ jobs:
       - name: Running translation tests
         run: pipenv run python -m scripts.extract_translation_templates --test
       - name: Python linting
-        run: pipenv run ./scripts/run_lint_python.sh
+        run: make lint-python
       - name: Get yarn cache
         id: get-yarn-cache
         run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
@@ -88,7 +88,7 @@ jobs:
       - name: Functional tests spec lint
         run: ./scripts/lint_functional_test_specs.sh
       - name: Javascript linting
-        run: yarn lint
+        run: make lint-js
   test-unit:
     needs: python-dependencies
     runs-on: ubuntu-22.04
@@ -121,15 +121,15 @@ jobs:
       - name: Link env vars
         run: ln -sf .development.env .env
       - name: Running unit tests
-        run: pipenv run ./scripts/run_tests_unit.sh
+        run: make test-unit
   validate-schemas:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
       - name: Run validator
         run: ./scripts/run_validator.sh
       - name: Running schema tests
-        run: ./scripts/validate_test_schemas.sh
+        run: make validate-test-schemas
   test-functional:
     needs: [python-dependencies, node-dependencies]
     strategy:
@@ -169,7 +169,7 @@ jobs:
       - name: Docker compose
         run: docker-compose --version && RUNNER_ENV_FILE=.functional-tests.env docker-compose up --build -d
       - name: Functional tests
-        run: ./scripts/run_tests_functional.sh ${{ matrix.suite }}
+        run: make test-functional-suite SUITE=${{ matrix.suite }}
       - name: Docker compose shutdown
         run: RUNNER_ENV_FILE=.functional-tests.env docker-compose kill
   docker-push:

.prettierignore

@@ -1 +1,2 @@
-tests/functional/generated_pages/
+# Exclude generated_pages directory
+tests/functional/generated_pages/

Makefile

@@ -1,6 +1,7 @@
 SCHEMAS_VERSION=`cat .schemas-version`
 DESIGN_SYSTEM_VERSION=`cat .design-system-version`
 RUNNER_ENV_FILE?=.development.env
+SCHEMA_PATH=./schemas/test/en/
 
 clean:
 	find schemas/* -prune | grep -v "schemas/test" | xargs rm -r
@@ -16,14 +17,17 @@ load-design-system-templates:
 
 build: load-design-system-templates load-schemas translate
 
-lint: lint-python
-	yarn lint
+generate-pages:
+	yarn generate_pages
+
+lint: lint-python lint-js
 
 lint-python:
 	pipenv run ./scripts/run_lint_python.sh
 
-format: format-python
-	yarn format
+lint-test-python: lint-python test-unit
+
+format: format-python format-js
 
 format-python:
 	pipenv run isort .
@@ -35,11 +39,32 @@ test:
 test-unit:
 	pipenv run ./scripts/run_tests_unit.sh
 
-test-functional:
-	pipenv run ./scripts/run_tests_functional.sh
+test-functional: generate-pages
+	yarn test_functional
+
+test-functional-headless: generate-pages
+	EQ_RUN_FUNCTIONAL_TESTS_HEADLESS='True' make test-functional
+
+test-functional-spec: generate-pages
+	yarn test_functional --spec ./tests/functional/spec/$(SPEC)
+
+test-functional-suite: generate-pages
+	yarn test_functional --suite $(SUITE)
+
+lint-js:
+	yarn lint
+
+format-js:
+	yarn format
+
+generate-spec:
+	pipenv run python -m tests.functional.generate_pages schemas/test/en/$(SCHEMA).json ./tests/functional/generated_pages/$(patsubst test_%,%,$(SCHEMA)) -r '../../base_pages' -s tests/functional/spec/$(SCHEMA).spec.js
 
 validate-test-schemas:
-	pipenv run ./scripts/validate_test_schemas.sh
+	./scripts/validate_test_schemas.sh
+
+validate-test-schema:
+	./scripts/validate_test_schemas.sh $(SCHEMA_PATH)$(SCHEMA).json
 
 translation-templates:
 	pipenv run python -m scripts.extract_translation_templates

README.md

@@ -217,11 +217,11 @@ yarn
 Available commands:
 
 | Command                | Task                                                                                                      |
-| ---------------------- | --------------------------------------------------------------------------------------------------------- |
-| `yarn test_functional` | Runs the functional tests through Webdriver (requires app running on localhost:5000 and generated pages). |
-| `yarn generate_pages`  | Generates the functional test pages.                                                                      |
-| `yarn lint`            | Lints the JS, reporting errors/warnings.                                                                  |
-| `yarn format`          | Format the json schemas.                                                                                  |
+|------------------------| --------------------------------------------------------------------------------------------------------- |
+| `make test-functional` | Runs the functional tests through Webdriver (requires app running on localhost:5000 and generated pages). |
+| `make generate-pages`  | Generates the functional test pages.                                                                      |
+| `make lint-js`         | Lints the JS, reporting errors/warnings.                                                                  |
+| `make format-js`       | Format the json schemas.                                                                                  |
 
 ---
 
@@ -241,44 +241,40 @@ RUNNER_ENV_FILE=.functional-tests.env make run
 
 This will set the correct environment variables for running the functional tests.
 
-Then you can run:
+Then you can run either:
 
 ``` shell
 make test-functional
 ```
-
-This will delete the `tests/functional/generated_pages` directory and regenerate all the files in it from the schemas.
-
-You can also individually run the `generate_pages` and `test_functional` yarn scripts:
+or
 
 ``` shell
-yarn generate_pages
-yarn test_functional
+make test-functional-headless
 ```
 
+This will delete the `tests/functional/generated_pages` directory and regenerate all the files in it from the schemas.
 
 To generate the pages manually you can run the `generate_pages` scripts with the schema directory. Run it from the `tests/functional` directory as follows:
 
 ``` shell
 ./generate_pages.py ../../schemas/test/en/ ./generated_pages -r "../../base_pages"
 ```
 
-To generate a spec file with the imports included, you can use the `generate_pages.py` script on a single schema with the `-s` argument.
-
+To generate a spec file with the imports included, you can pass the schema name as an argument without the file extension, e.g. `SCHEMA=test_address`:
 ``` shell
-./generate_pages.py ../../schemas/test/en/test_multiple_piping.json ./temp_directory -r "../../base_pages" -s spec/test_multiple_piping.spec.js
+make generate-spec SCHEMA=<schema-name>
 ```
 
 If you have already built the generated pages, then the functional tests can be executed with:
 
 ``` shell
-yarn test_functional
+make test-functional
 ```
 
-This can be limited to a single spec using:
+This can be limited to a single spec where argument needed is the remainder of the path after `./tests/functional/spec/` (which is included in the command):
 
 ``` shell
-yarn test_functional --spec ./tests/functional/spec/exit.spec.js
+make test-functional-spec SPEC=<spec>
 ```
 
 To run a single test, add `.only` into the name of any `describe` or `it` function:
@@ -288,10 +284,16 @@ To run a single test, add `.only` into the name of any `describe` or `it` functi
 `it.only('Given this is a test', function() {...}`
 
 Test suites are configured in the `wdio.conf.js` file.
-An individual test suite can be run using:
+An individual test suite can be run using the suite names as the argument to this command. The suites that can be used with command below are:
+* timeout_modal_expired
+* timeout_modal_extended
+* timeout_modal_extended_new_window
+* features
+* general
+* components
 
 ``` shell
-yarn test_functional --suite <suite>
+make test-functional-suite SUITE=<suite>
 ```
 
 To run the tests against a remote deployment you will need to specify the environment variable of EQ_FUNCTIONAL_TEST_ENV eg:

package.json

@@ -15,7 +15,7 @@
         "lint": "yarn generate_pages && yarn lint:tests && yarn lint:test-schemas",
         "lint:tests": "prettier --check \"tests/functional/**/*.js\" && eslint \"tests/functional/**/*.js\"",
         "lint:test-schemas": "prettier --check \"schemas/test/*/*.json\" && eslint \"schemas/test/**/*.json\"",
-        "test_functional": "./node_modules/.bin/wdio tests/functional/wdio.conf.js --suite $1",
+        "test_functional": "./node_modules/.bin/wdio tests/functional/wdio.conf.js",
         "generate_pages": "rm -rf ./tests/functional/generated_pages && pipenv run python -m tests.functional.generate_pages schemas/test/en/ ./tests/functional/generated_pages -r '../../base_pages'",
         "format": "yarn format:tests && yarn format:test-schemas",
         "format:tests": "prettier \"tests/functional/**/*.js\" --write && eslint --fix \"tests/functional/**/*.js\"",

setup.cfg

@@ -6,3 +6,8 @@ filterwarnings=
     ignore:.*formatargspec.*:DeprecationWarning
     ignore:.*isAlive.*:PendingDeprecationWarning
 
+[flake8]
+# Ignore node_modules and cloned repos when not in a virtual environment
+exclude = node_modules/*,tests/*,src/*
+max-line-length = 160
+ignore = C815,C816,W503,E203