Split unit and integration tests.

Signed-off-by: dblock <[email protected]>

tools/src/tester/start.ts

@@ -32,6 +32,7 @@ const options = {
 
 // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
 process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
+
 const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge()
 const runner = new TestsRunner(spec, opts.testsPath, options)
 void runner.run().then(() => { _.noop() })

tools/tests/tester/integration/StoryEvaluator.test.ts

@@ -0,0 +1,55 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
+import { read_yaml } from '../../../helpers'
+import { type OpenAPIV3 } from 'openapi-types'
+
+describe('Story Evaluator', () => {
+  beforeAll(() => {
+    const spec = read_yaml('tools/tests/tester/integration/fixtures/specs/indices_excerpt.yaml')
+    create_shared_resources(spec as OpenAPIV3.Document)  
+  })
+
+  test('passed', async () => {
+    const actual = await load_actual_evaluation('passed')
+    const expected = await load_expected_evaluation('passed')
+    expect(actual).toEqual(expected)
+  })
+
+  test('skipped', async () => {
+    const actual = await load_actual_evaluation('skipped')
+    const expected = await load_expected_evaluation('skipped')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/not_found', async () => {
+    const actual = await load_actual_evaluation('failed/not_found')
+    const expected = await load_expected_evaluation('failed/not_found')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/invalid_data', async () => {
+    const actual = await load_actual_evaluation('failed/invalid_data')
+    const expected = await load_expected_evaluation('failed/invalid_data')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/prologue_error', async () => {
+    const actual = await load_actual_evaluation('error/prologue_error')
+    const expected = await load_expected_evaluation('error/prologue_error')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/chapter_error', async () => {
+    const actual = await load_actual_evaluation('error/chapter_error')
+    const expected = await load_expected_evaluation('error/chapter_error')
+    expect(actual).toEqual(expected)
+  })
+})

tools/tests/tester/TestsRunner.test.ts → tools/tests/tester/integration/TestsRunner.test.ts

@@ -7,16 +7,18 @@
 * compatible open source license.
 */
 
-import { read_yaml } from '../../helpers'
-import TestsRunner from '../../src/tester/TestsRunner'
+import { read_yaml } from '../../../helpers'
+import TestsRunner from '../../../src/tester/TestsRunner'
 import { type OpenAPIV3 } from 'openapi-types'
 import { load_expected_evaluation, scrub_errors } from './helpers'
 
 test('stories folder', async () => {
   // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
   process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
-  const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
-  const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
+  const spec = read_yaml('tools/tests/tester/integration/fixtures/specs/indices_excerpt.yaml')
+  const runner = new TestsRunner(
+    spec as OpenAPIV3.Document,
+    'tools/tests/tester/integration/fixtures/stories', {})
   const actual_evaluations = await runner.run(true) as any[]
   for (const evaluation of actual_evaluations) scrub_errors(evaluation)
   for (const evaluation of actual_evaluations) {

tools/tests/tester/fixtures/evals/error/chapter_error.yaml → tools/tests/tester/integration/fixtures/evals/error/chapter_error.yaml

@@ -1,5 +1,5 @@
 display_path: error/chapter_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml
+full_path: tools/tests/tester/integration/fixtures/stories/error/chapter_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/error/prologue_error.yaml → tools/tests/tester/integration/fixtures/evals/error/prologue_error.yaml

@@ -1,5 +1,5 @@
 display_path: error/prologue_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
+full_path: tools/tests/tester/integration/fixtures/stories/error/prologue_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/failed/invalid_data.yaml → tools/tests/tester/integration/fixtures/evals/failed/invalid_data.yaml

@@ -1,5 +1,5 @@
 display_path: failed/invalid_data.yaml
-full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
+full_path: tools/tests/tester/integration/fixtures/stories/failed/invalid_data.yaml
 
 result: FAILED
 description: This story should failed due invalid data.

tools/tests/tester/fixtures/evals/failed/not_found.yaml → tools/tests/tester/integration/fixtures/evals/failed/not_found.yaml

@@ -1,5 +1,5 @@
 display_path: failed/not_found.yaml
-full_path: tools/tests/tester/fixtures/stories/failed/not_found.yaml
+full_path: tools/tests/tester/integration/fixtures/stories/failed/not_found.yaml
 
 result: FAILED
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/passed.yaml → tools/tests/tester/integration/fixtures/evals/passed.yaml

@@ -1,5 +1,5 @@
 display_path: passed.yaml
-full_path: tools/tests/tester/fixtures/stories/passed.yaml
+full_path: tools/tests/tester/integration/fixtures/stories/passed.yaml
 
 result: PASSED
 description: This story should pass.

tools/tests/tester/fixtures/evals/skipped.yaml → tools/tests/tester/integration/fixtures/evals/skipped.yaml

@@ -1,5 +1,5 @@
 display_path: skipped.yaml
-full_path: "tools/tests/tester/fixtures/stories/skipped.yaml"
+full_path: "tools/tests/tester/integration/fixtures/stories/skipped.yaml"
 
 result: SKIPPED
 description: This story should be skipped.

tools/tests/tester/helpers.ts → tools/tests/tester/integration/helpers.ts

@@ -7,16 +7,16 @@
 * compatible open source license.
 */
 
-import ChapterReader from '../../src/tester/ChapterReader'
-import SpecParser from '../../src/tester/SpecParser'
-import SchemaValidator from '../../src/tester/SchemaValidator'
-import SharedResources from '../../src/tester/SharedResources'
+import ChapterReader from '../../../src/tester/ChapterReader'
+import SpecParser from '../../../src/tester/SpecParser'
+import SchemaValidator from '../../../src/tester/SchemaValidator'
+import SharedResources from '../../../src/tester/SharedResources'
 import { type OpenAPIV3 } from 'openapi-types'
 import YAML from 'yaml'
-import type { StoryEvaluation } from '../../src/tester/types/eval.types'
-import type { Story } from '../../src/tester/types/story.types'
-import { read_yaml } from '../../helpers'
-import StoryEvaluator from '../../src/tester/StoryEvaluator'
+import type { StoryEvaluation } from '../../../src/tester/types/eval.types'
+import type { Story } from '../../../src/tester/types/story.types'
+import { read_yaml } from '../../../helpers'
+import StoryEvaluator from '../../../src/tester/StoryEvaluator'
 
 export function create_shared_resources (spec: any): void {
   // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
@@ -34,21 +34,21 @@ export function print_yaml (obj: any): void {
 export function scrub_errors (obj: any): void {
   for (const key in obj) {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-    if (key === 'error') obj[key] = obj[key].message
+    if (key === 'error') obj[key] = obj[key]?.message
     else if (typeof obj[key] === 'object') scrub_errors(obj[key])
   }
 }
 
 export async function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Promise<any> {
-  const expected = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`)
+  const expected = read_yaml(`tools/tests/tester/integration/fixtures/evals/${name}.yaml`)
   if (exclude_full_path) delete expected.full_path
   return expected
 }
 
 export async function load_actual_evaluation (name: string): Promise<StoryEvaluation> {
-  const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`)
+  const story: Story = read_yaml(`tools/tests/tester/integration/fixtures/stories/${name}.yaml`)
   const display_path = `${name}.yaml`
-  const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml`
+  const full_path = `tools/tests/tester/integration/fixtures/stories/${name}.yaml`
   const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate()
   scrub_errors(actual)
   return actual

tools/tests/tester/overall_result.test.ts → tools/tests/tester/integration/overall_result.test.ts

@@ -7,8 +7,8 @@
 * compatible open source license.
 */
 
-import { overall_result } from '../../src/tester/helpers'
-import { type Evaluation, Result } from '../../src/tester/types/eval.types'
+import { overall_result } from '../../../src/tester/helpers'
+import { type Evaluation, Result } from '../../../src/tester/types/eval.types'
 
 function e (...results: Result[]): Evaluation[] {
   return results.map(result => ({ result }))

tools/tests/tester/ansi.test.ts → tools/tests/tester/unit/ansi.test.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import * as ansi from '../../src/tester/Ansi'
+import * as ansi from '../../../src/tester/Ansi'
 
 test('b', () => {
   expect(ansi.b('xyz')).toEqual('\x1b[1mxyz\x1b[0m')

tools/tests/tester/start.test.ts → tools/tests/tester/unit/start.test.ts

@@ -8,7 +8,7 @@
 */
 
 import { spawnSync } from 'child_process'
-import * as ansi from '../../src/tester/Ansi'
+import * as ansi from '../../../src/tester/Ansi'
 import * as path from 'path'
 
 const spec = (args: string[]): any => {
@@ -30,28 +30,28 @@ test('--invalid', async () => {
 })
 
 test('displays story filename', async () => {
-  expect(spec(['--tests', 'tools/tests/tester/fixtures/empty_story']).stdout).toContain(
+  expect(spec(['--tests', 'tools/tests/tester/unit/fixtures/empty_story']).stdout).toContain(
     `${ansi.green('PASSED ')} ${ansi.cyan(ansi.b('empty.yaml'))}`
   )
 })
 
 test('displays story description', async () => {
-  expect(spec(['--tests', 'tools/tests/tester/fixtures/empty_with_description.yaml']).stdout).toContain(
+  expect(spec(['--tests', 'tools/tests/tester/unit/fixtures/empty_with_description.yaml']).stdout).toContain(
     `${ansi.green('PASSED ')} ${ansi.cyan(ansi.b('A story with no beginning or end.'))}`
   )
 })
 
 test.todo('--tab-width')
 
 test('--dry-run', async () => {
-  const test_yaml = 'tools/tests/tester/fixtures/empty_with_all_the_parts.yaml'
+  const test_yaml = 'tools/tests/tester/unit/fixtures/empty_with_all_the_parts.yaml'
   const s = spec(['--dry-run', '--tests', test_yaml]).stdout
-  const full_path = path.join(__dirname, '../../../' + test_yaml)
+  const full_path = path.join(__dirname, '../../../../' + test_yaml)
   expect(s).toEqual(`${ansi.yellow('SKIPPED')} ${ansi.cyan(ansi.b('A story with all its parts.'))} ${ansi.gray('(' + full_path + ')')}\n\n\n`)
 })
 
 test('--dry-run --verbose', async () => {
-  const s = spec(['--dry-run', '--verbose', '--tests', 'tools/tests/tester/fixtures/empty_with_all_the_parts.yaml']).stdout
+  const s = spec(['--dry-run', '--verbose', '--tests', 'tools/tests/tester/unit/fixtures/empty_with_all_the_parts.yaml']).stdout
   expect(s).toContain(`${ansi.yellow('SKIPPED')} ${ansi.cyan(ansi.b('A story with all its parts.'))}`)
   expect(s).toContain(`${ansi.yellow('SKIPPED')} CHAPTERS`)
   expect(s).toContain(`${ansi.yellow('SKIPPED')} EPILOGUES`)