From f04409147fe1c192c6b5799fc18462d14e4d29b4 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Mon, 24 Aug 2020 18:31:02 +0200 Subject: [PATCH 1/9] feat: init js translator Only domain step is supported for now --- src/lib/translators/index.ts | 2 ++ src/lib/translators/js.ts | 70 ++++++++++++++++++++++++++++++++++++ tests/unit/js.spec.ts | 38 ++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 src/lib/translators/js.ts create mode 100644 tests/unit/js.spec.ts diff --git a/src/lib/translators/index.ts b/src/lib/translators/index.ts index 41000da887..f5d8ed01aa 100644 --- a/src/lib/translators/index.ts +++ b/src/lib/translators/index.ts @@ -8,6 +8,7 @@ import { PipelineStepName } from '@/lib/steps'; import { BaseTranslator } from './base'; +import { JavaScriptTranslator } from './js'; import { Mongo36Translator } from './mongo'; import { Mongo40Translator } from './mongo4'; @@ -57,5 +58,6 @@ export function availableTranslators() { return { ...TRANSLATORS }; } +registerTranslator('js', JavaScriptTranslator); registerTranslator('mongo36', Mongo36Translator); registerTranslator('mongo40', Mongo40Translator); diff --git a/src/lib/translators/js.ts b/src/lib/translators/js.ts new file mode 100644 index 0000000000..08580302eb --- /dev/null +++ b/src/lib/translators/js.ts @@ -0,0 +1,70 @@ +/** + * The JavaScript translator turns pipelines steps into functions. + * + * With it, pipelines can be executed right in the browser, without any + * database engine. This is very useful for testing, getting an idea of how + * transformations works and provide a specification for transformations steps. + * For web apps, it can also enable offline data processing. + * + * Input data must be of the form: + * ``` + * { + * "domainA": [{ + * "column1": 123, + * "column2": 456, + * ... + * }, ...], + * "domainB": [...], + * ... + * } + * ``` + */ +import * as S from '@/lib/steps'; +import { BaseTranslator } from '@/lib/translators/base'; + +type ColumnName = string | number; +type DataRow = Record; +export type DataTable = DataRow[]; +export type DataDomains = Record; + +/** + * Each step will be translated into a function that outputs a data table. We call these StepFunctions. + * + * Some step functions needs the entire library of data domains as argument (domain and combination steps), + * but most only needs the current data table. They all have the same signature so they can be called easily in a loop. + */ +type JsStepFunction = (data: Readonly, domains: Readonly) => DataTable; + +export class JavaScriptTranslator extends BaseTranslator { + static label = 'JavaScript'; + + constructor() { + super(); + } + + translate(pipeline: S.Pipeline): JsStepFunction[] { + return super.translate(pipeline) as JsStepFunction[]; + } + + // transform a "domain" step into corresponding function + domain(domainStep: Readonly): JsStepFunction { + return function(_data: Readonly, dataDomains: Readonly) { + return dataDomains[domainStep.domain]; + }; + } +} + +/** + * The execute function provides a simple way to execute all the translated JsStepFunctions sequentially, to return the + * result of the data transformation. + */ +export function execute( + dataDomains: Readonly, + jsStepFunctions: Readonly, +): DataTable { + let result: DataTable = []; + for (const jsStepFunction of jsStepFunctions) { + result = jsStepFunction(result, dataDomains); + } + return result; +} diff --git a/tests/unit/js.spec.ts b/tests/unit/js.spec.ts new file mode 100644 index 0000000000..a8965e4775 --- /dev/null +++ b/tests/unit/js.spec.ts @@ -0,0 +1,38 @@ +import { getTranslator } from '@/lib/translators'; +import {execute, JavaScriptTranslator, DataTable, DataDomains} from '@/lib/translators/js'; +import {Pipeline} from "@/lib/steps"; + +describe('Pipeline to js function translator', () => { + const jsTranslator = getTranslator('js') as JavaScriptTranslator; + const SAMPLE_DATA: DataDomains = { + domainA: [ + {label: 'A', value: 1}, + {label: 'B', value: 2} + ], + domainB: [ + {label: 'alpha', value: 0.1}, + {label: 'beta', value: 0.2} + ] + } + + function expectResultFromPipeline(pipeline: Pipeline, data: DataDomains, expectedResult: DataTable) { + expect(execute(data, jsTranslator.translate(pipeline))).toEqual(expectedResult); + } + + it('can handle domain steps', () => { + expectResultFromPipeline( + [{ name: 'domain', domain: 'domainA' }], + SAMPLE_DATA, + [ + {label: 'A', value: 1}, + {label: 'B', value: 2} + ]); + expectResultFromPipeline( + [{ name: 'domain', domain: 'domainB' }], + SAMPLE_DATA, + [ + {label: 'alpha', value: 0.1}, + {label: 'beta', value: 0.2} + ]); + }); +}); From e1ba7c1ada4f2cf8da4105b13480daab16138b86 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Thu, 27 Aug 2020 14:56:27 +0200 Subject: [PATCH 2/9] feat: filter step for js translator --- src/lib/steps.ts | 4 +- src/lib/translators/js.ts | 58 +++++++++ tests/unit/js.spec.ts | 249 ++++++++++++++++++++++++++++++++++---- 3 files changed, 287 insertions(+), 24 deletions(-) diff --git a/src/lib/steps.ts b/src/lib/steps.ts index ad73cf9366..4a433ab2b7 100644 --- a/src/lib/steps.ts +++ b/src/lib/steps.ts @@ -156,13 +156,13 @@ export type FilterSimpleCondition = | FilterConditionEquality | FilterConditionInclusion; -type FilterConditionComparison = { +export type FilterConditionComparison = { column: string; value: number | string; operator: 'gt' | 'ge' | 'lt' | 'le'; }; -type FilterConditionEquality = { +export type FilterConditionEquality = { column: string; value: any; operator: 'eq' | 'ne' | 'isnull' | 'notnull' | 'matches' | 'notmatches'; diff --git a/src/lib/translators/js.ts b/src/lib/translators/js.ts index 08580302eb..1e41dd9dec 100644 --- a/src/lib/translators/js.ts +++ b/src/lib/translators/js.ts @@ -52,6 +52,64 @@ export class JavaScriptTranslator extends BaseTranslator { return dataDomains[domainStep.domain]; }; } + + // transform a "filter" step into corresponding function + filter(filterStep: Readonly): JsStepFunction { + type FilteringFunction = (dataRow: DataRow) => boolean; + + function filteringFunctionForSimpleCondition( + condition: S.FilterSimpleCondition, + ): FilteringFunction { + switch (condition.operator) { + case 'eq': + return d => d[condition.column] == condition.value; + case 'ne': + return d => d[condition.column] != condition.value; + case 'lt': + return d => d[condition.column] < condition.value; + case 'le': + return d => d[condition.column] <= condition.value; + case 'gt': + return d => d[condition.column] > condition.value; + case 'ge': + return d => d[condition.column] >= condition.value; + case 'in': + return d => condition.value.includes(d[condition.column]); + case 'nin': + return d => !condition.value.includes(d[condition.column]); + case 'isnull': + return d => d[condition.column] == null; + case 'notnull': + return d => d[condition.column] != null; + case 'matches': + // eslint-disable-next-line no-case-declarations + const regexToMatch = new RegExp(condition.value); + return d => regexToMatch.test(d[condition.column]); + case 'notmatches': + // eslint-disable-next-line no-case-declarations + const regexToNotMatch = new RegExp(condition.value); + return d => !regexToNotMatch.test(d[condition.column]); + default: + throw new Error(`Invalid condition operator in ${JSON.stringify(condition, null, 2)}`); + } + } + + function filteringFunctionForCondition(condition: S.FilterCondition): FilteringFunction { + if (S.isFilterComboAnd(condition)) { + const filteringFunctions = condition.and.map(filteringFunctionForCondition); + return d => filteringFunctions.every(f => f(d)); + } else if (S.isFilterComboOr(condition)) { + const filteringFunctions = condition.or.map(filteringFunctionForCondition); + return d => filteringFunctions.some(f => f(d)); + } else { + return filteringFunctionForSimpleCondition(condition); + } + } + + return function(data: Readonly, _dataDomains: Readonly) { + return data.filter(filteringFunctionForCondition(filterStep.condition)); + }; + } } /** diff --git a/tests/unit/js.spec.ts b/tests/unit/js.spec.ts index a8965e4775..19a3580a7e 100644 --- a/tests/unit/js.spec.ts +++ b/tests/unit/js.spec.ts @@ -1,38 +1,243 @@ +import { Pipeline } from '@/lib/steps'; import { getTranslator } from '@/lib/translators'; -import {execute, JavaScriptTranslator, DataTable, DataDomains} from '@/lib/translators/js'; -import {Pipeline} from "@/lib/steps"; +import { DataDomains, DataTable, execute, JavaScriptTranslator } from '@/lib/translators/js'; describe('Pipeline to js function translator', () => { const jsTranslator = getTranslator('js') as JavaScriptTranslator; const SAMPLE_DATA: DataDomains = { domainA: [ - {label: 'A', value: 1}, - {label: 'B', value: 2} + { label: 'A', value: 1 }, + { label: 'B', value: 2 }, ], domainB: [ - {label: 'alpha', value: 0.1}, - {label: 'beta', value: 0.2} - ] - } + { label: 'alpha', value: 0.1 }, + { label: 'beta', value: 0.2 }, + ], + }; - function expectResultFromPipeline(pipeline: Pipeline, data: DataDomains, expectedResult: DataTable) { + function expectResultFromPipeline( + pipeline: Pipeline, + data: DataDomains, + expectedResult: DataTable, + ) { expect(execute(data, jsTranslator.translate(pipeline))).toEqual(expectedResult); } - it('can handle domain steps', () => { - expectResultFromPipeline( - [{ name: 'domain', domain: 'domainA' }], - SAMPLE_DATA, - [ - {label: 'A', value: 1}, - {label: 'B', value: 2} + it('should handle domain steps', () => { + expectResultFromPipeline([{ name: 'domain', domain: 'domainA' }], SAMPLE_DATA, [ + { label: 'A', value: 1 }, + { label: 'B', value: 2 }, ]); - expectResultFromPipeline( - [{ name: 'domain', domain: 'domainB' }], - SAMPLE_DATA, - [ - {label: 'alpha', value: 0.1}, - {label: 'beta', value: 0.2} + expectResultFromPipeline([{ name: 'domain', domain: 'domainB' }], SAMPLE_DATA, [ + { label: 'alpha', value: 0.1 }, + { label: 'beta', value: 0.2 }, ]); }); + + describe('filter step', () => { + const SAMPLE_DATA = [ + { label: 'A', value: 1 }, + { label: 'B', value: 2 }, + { label: 'alpha', value: 0.1 }, + { label: 'beta', value: 0.2 }, + ]; + + const DATA_WITH_NULL_LABELS = [ + { + label: 'A', + }, + { + label: 'B', + }, + { + label: null, + }, + { + label: undefined, + }, + { + value: 2, + }, + ]; + + it('should handle simple equality condition', () => { + const filterALabel = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + value: 'A', + operator: 'eq', + }, + }); + expect(filterALabel(SAMPLE_DATA, {})).toEqual([{label: 'A', value: 1}]); + }); + + it('should handle and conditions with matches and gt', () => { + const filterValuesGreaterThan1AndLabelsWithOnlyOneLetter = jsTranslator.filter({ + name: 'filter', + condition: { + and: [ + { + column: 'label', + value: '^[A-Za-z]$', + operator: 'matches', + }, + { + column: 'value', + value: 1, + operator: 'gt', + }, + ], + }, + }); + expect(filterValuesGreaterThan1AndLabelsWithOnlyOneLetter(SAMPLE_DATA, {})).toEqual([ + {label: 'B', value: 2}, + ]); + }); + + it('should handle or conditions with notmatches and le', () => { + const filterLabelsWithMoreThanOneLetterOrValuesLessThan1 = jsTranslator.filter({ + name: 'filter', + condition: { + or: [ + { + column: 'label', + value: '^[A-Za-z]$', + operator: 'notmatches', + }, + { + column: 'value', + value: 1, + operator: 'le', + }, + ], + }, + }); + expect(filterLabelsWithMoreThanOneLetterOrValuesLessThan1(SAMPLE_DATA, {})).toEqual([ + {label: 'A', value: 1}, + {label: 'alpha', value: 0.1}, + {label: 'beta', value: 0.2}, + ]); + }); + + it('should handle notnull conditions', () => { + const filterNotNull = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + operator: 'notnull', + value: null, + }, + }); + expect(filterNotNull(DATA_WITH_NULL_LABELS, {})).toEqual([ + { + label: 'A', + }, + { + label: 'B', + }, + ]); + }); + + it('should handle null conditions', () => { + const filterNull = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + operator: 'isnull', + value: null, + }, + }); + expect(filterNull(DATA_WITH_NULL_LABELS, {})).toEqual([ + { + label: null, + }, + { + label: undefined, + }, + { + value: 2, + }, + ]); + }); + + it('should handle ne conditions', () => { + const filterNotEqual = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + operator: 'ne', + value: 'A', + }, + }); + expect(filterNotEqual(SAMPLE_DATA, {})).toEqual([ + {label: 'B', value: 2}, + {label: 'alpha', value: 0.1}, + {label: 'beta', value: 0.2}, + ]); + }); + + it('should handle or conditions with lt and ge', () => { + const filterLowerThan1OrGreatherThan1 = jsTranslator.filter({ + name: 'filter', + condition: { + or: [ + {column: 'value', operator: 'lt', value: 0.2}, + { + column: 'value', + operator: 'ge', + value: 2, + }, + ], + }, + }); + expect(filterLowerThan1OrGreatherThan1(SAMPLE_DATA, {})).toEqual([ + {label: 'B', value: 2}, + {label: 'alpha', value: 0.1}, + ]); + }); + + it('should handle in conditions', () => { + const filterIn = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + operator: 'in', + value: ['A', 'alpha'], + }, + }); + expect(filterIn(SAMPLE_DATA, {})).toEqual([ + {label: 'A', value: 1}, + {label: 'alpha', value: 0.1}, + ]); + }); + + it('should handle nin conditions', () => { + const filterNotIn = jsTranslator.filter({ + name: 'filter', + condition: { + column: 'label', + operator: 'nin', + value: ['A', 'alpha'], + }, + }); + expect(filterNotIn(SAMPLE_DATA, {})).toEqual([ + { label: 'B', value: 2 }, + { label: 'beta', value: 0.2 }, + ]); + }); + + it('should throw if the operator is not valid', () => { + const invalidFilter = jsTranslator.filter({ + name: 'filter', + // eslint-disable-next-line @typescript-eslint/ban-ts-ignore + // @ts-ignore + condition: { + column: 'label', + operator: 'what', + }, + }); + expect(() => invalidFilter(SAMPLE_DATA, {})).toThrow(); + }); + }); }); From 3deb3c0cec954581d5a6e4102936cf755fdb149b Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 02:20:47 +0200 Subject: [PATCH 3/9] WIP Prototype usage of js translator in playground --- playground/app.js | 106 +++++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/playground/app.js b/playground/app.js index 5fb8a3a220..0d062efc3b 100644 --- a/playground/app.js +++ b/playground/app.js @@ -12,9 +12,8 @@ const { setCodeEditor, } = vqb; -const TRANSLATOR = 'mongo40'; - -const mongo40translator = getTranslator(TRANSLATOR); +const mongo40translator = getTranslator('mongo40'); +const jsTranslator = getTranslator('js'); // Example to set a custom editor: // This one is quite simple. It customizes the placeholder. @@ -165,10 +164,55 @@ class MongoService { const mongoservice = new MongoService(); const mongoBackendPlugin = servicePluginFactory(mongoservice); +class DataService { + constructor() { + this.dataDomains = { + defaultDataset: [{ + label: 'A', + value: 1, + },{ + label: 'B', + value: 2, + }], + sales: [{ + label: 'bli', + value: 1, + },{ + label: 'blu', + value: 2, + }] + }; + } + + async listCollections(_store) { + return Object.keys(this.dataDomains); + } + + async executePipeline(_store, pipeline, limit, offset = 0) { + const jsStepFunctions = jsTranslator.translate(pipeline); + try { + let result = []; + for (const jsStepFunction of jsStepFunctions) { + result = jsStepFunction(result, this.dataDomains); + } + + let dataset = mongoResultsToDataset(result); + return { data: dataset }; + } catch (e) { + return { + error: [{type: 'error', message: e.message || e}], + }; + } + } +} + +const dataService = new DataService(); +const dataBackendPlugin = servicePluginFactory(dataService); + async function buildVueApp() { Vue.use(Vuex); const store = new Vuex.Store({ - plugins: [mongoBackendPlugin], + plugins: [dataBackendPlugin], }); new Vue({ @@ -187,51 +231,9 @@ async function buildVueApp() { }, created: async function() { registerModule(this.$store, { - currentPipelineName: 'pipeline', + currentPipelineName: 'sample_pipeline', pipelines: { - pipeline: [ - { - name: 'domain', - domain: 'sales', - }, - { - name: 'filter', - condition: { - column: 'Price', - operator: 'ge', - value: 1200, - }, - } - ], - pipelineAmex: [ - { - name: 'domain', - domain: 'sales', - }, - { - name: 'filter', - condition: { - column: 'Payment_Type', - operator: 'eq', - value: 'Amex', - }, - } - ], - pipelineVisa: [ - { - name: 'domain', - domain: 'sales', - }, - { - name: 'filter', - condition: { - column: 'Payment_Type', - operator: 'eq', - value: 'Visa', - }, - } - ], - pipelineMastercard: [ + sample_pipeline: [ { name: 'domain', domain: 'sales', @@ -239,15 +241,15 @@ async function buildVueApp() { { name: 'filter', condition: { - column: 'Payment_Type', - operator: 'eq', - value: 'Mastercard', + column: 'label', + operator: 'notnull', + value: null, }, } ], }, currentDomain: 'sales', - translator: TRANSLATOR, + translator: 'js', // use lodash interpolate interpolateFunc: (value, context) => _.template(value)(context), variables: { From 0b663da481005dc1900c4497c315210d1a1ca4a8 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 02:32:01 +0200 Subject: [PATCH 4/9] style: lint js translator tests --- tests/unit/js.spec.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/unit/js.spec.ts b/tests/unit/js.spec.ts index 19a3580a7e..7db9f7473f 100644 --- a/tests/unit/js.spec.ts +++ b/tests/unit/js.spec.ts @@ -69,7 +69,7 @@ describe('Pipeline to js function translator', () => { operator: 'eq', }, }); - expect(filterALabel(SAMPLE_DATA, {})).toEqual([{label: 'A', value: 1}]); + expect(filterALabel(SAMPLE_DATA, {})).toEqual([{ label: 'A', value: 1 }]); }); it('should handle and conditions with matches and gt', () => { @@ -91,7 +91,7 @@ describe('Pipeline to js function translator', () => { }, }); expect(filterValuesGreaterThan1AndLabelsWithOnlyOneLetter(SAMPLE_DATA, {})).toEqual([ - {label: 'B', value: 2}, + { label: 'B', value: 2 }, ]); }); @@ -114,9 +114,9 @@ describe('Pipeline to js function translator', () => { }, }); expect(filterLabelsWithMoreThanOneLetterOrValuesLessThan1(SAMPLE_DATA, {})).toEqual([ - {label: 'A', value: 1}, - {label: 'alpha', value: 0.1}, - {label: 'beta', value: 0.2}, + { label: 'A', value: 1 }, + { label: 'alpha', value: 0.1 }, + { label: 'beta', value: 0.2 }, ]); }); @@ -171,9 +171,9 @@ describe('Pipeline to js function translator', () => { }, }); expect(filterNotEqual(SAMPLE_DATA, {})).toEqual([ - {label: 'B', value: 2}, - {label: 'alpha', value: 0.1}, - {label: 'beta', value: 0.2}, + { label: 'B', value: 2 }, + { label: 'alpha', value: 0.1 }, + { label: 'beta', value: 0.2 }, ]); }); @@ -182,7 +182,7 @@ describe('Pipeline to js function translator', () => { name: 'filter', condition: { or: [ - {column: 'value', operator: 'lt', value: 0.2}, + { column: 'value', operator: 'lt', value: 0.2 }, { column: 'value', operator: 'ge', @@ -192,8 +192,8 @@ describe('Pipeline to js function translator', () => { }, }); expect(filterLowerThan1OrGreatherThan1(SAMPLE_DATA, {})).toEqual([ - {label: 'B', value: 2}, - {label: 'alpha', value: 0.1}, + { label: 'B', value: 2 }, + { label: 'alpha', value: 0.1 }, ]); }); @@ -207,8 +207,8 @@ describe('Pipeline to js function translator', () => { }, }); expect(filterIn(SAMPLE_DATA, {})).toEqual([ - {label: 'A', value: 1}, - {label: 'alpha', value: 0.1}, + { label: 'A', value: 1 }, + { label: 'alpha', value: 0.1 }, ]); }); From 026ac18212b475fd709daad75997f1d11c7cd4f5 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 02:32:29 +0200 Subject: [PATCH 5/9] feat: text step for js translator --- src/lib/translators/js.ts | 14 ++++++++++++++ tests/unit/js.spec.ts | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/lib/translators/js.ts b/src/lib/translators/js.ts index 1e41dd9dec..e875428bee 100644 --- a/src/lib/translators/js.ts +++ b/src/lib/translators/js.ts @@ -110,6 +110,20 @@ export class JavaScriptTranslator extends BaseTranslator { return data.filter(filteringFunctionForCondition(filterStep.condition)); }; } + + // transform a "text" step into corresponding function + text(step: Readonly): JsStepFunction { + function addTextColumnToRow(row: Readonly) { + return { + ...row, + [step.new_column]: step.text, + }; + } + + return function(data: Readonly, _dataDomains: Readonly) { + return data.map(addTextColumnToRow); + }; + } } /** diff --git a/tests/unit/js.spec.ts b/tests/unit/js.spec.ts index 7db9f7473f..374b63ac1f 100644 --- a/tests/unit/js.spec.ts +++ b/tests/unit/js.spec.ts @@ -240,4 +240,18 @@ describe('Pipeline to js function translator', () => { expect(() => invalidFilter(SAMPLE_DATA, {})).toThrow(); }); }); + + describe('text step', () => { + it('should add a column with text to each row', () => { + const addTextColumn = jsTranslator.text({ + name: 'text', + text: 'some text', + new_column: 'text_new_column', + }); + expect(addTextColumn(SAMPLE_DATA.domainA, {})).toEqual([ + { label: 'A', value: 1, text_new_column: 'some text' }, + { label: 'B', value: 2, text_new_column: 'some text' }, + ]); + }); + }); }); From ba7389f9dc1621bbe1110dc0105200fad3e4011c Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 03:05:01 +0200 Subject: [PATCH 6/9] feat: custom step for js translator --- src/lib/translators/js.ts | 13 +++++++++++++ tests/unit/js.spec.ts | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/lib/translators/js.ts b/src/lib/translators/js.ts index e875428bee..7848cc0d71 100644 --- a/src/lib/translators/js.ts +++ b/src/lib/translators/js.ts @@ -124,6 +124,19 @@ export class JavaScriptTranslator extends BaseTranslator { return data.map(addTextColumnToRow); }; } + + /** + * Transform a "custom" step into corresponding function + * + * The "query" parameter of the custom step should be a javascript function, of which the first parameter will be + * the data table, and the second an object of all available domains. + */ + custom(step: Readonly) { + const func = Function('"use strict";return (' + step.query + ')')(); + return function(data: Readonly, dataDomains: Readonly) { + return func(data, dataDomains); + }; + } } /** diff --git a/tests/unit/js.spec.ts b/tests/unit/js.spec.ts index 374b63ac1f..4cac4ffbd2 100644 --- a/tests/unit/js.spec.ts +++ b/tests/unit/js.spec.ts @@ -254,4 +254,23 @@ describe('Pipeline to js function translator', () => { ]); }); }); + + describe('custom step', () => { + it('should apply the custom function to data', () => { + const addPlipPloupAndDoubleValue = jsTranslator.custom({ + name: 'custom', + query: `function transformData(data) { + return data.map(d => ({ + ...d, + value: d.value * 2, + plip: "ploup" + })); + }`, + }); + expect(addPlipPloupAndDoubleValue(SAMPLE_DATA.domainA, {})).toEqual([ + { label: 'A', value: 2, plip: 'ploup' }, + { label: 'B', value: 4, plip: 'ploup' }, + ]); + }); + }); }); From f443f2d0c7d841b74d3569ad20f490d3cb2c90d6 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 10:58:00 +0200 Subject: [PATCH 7/9] fixup! WIP Prototype usage of js translator in playground --- playground/app.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/playground/app.js b/playground/app.js index 0d062efc3b..b6fb4bc66b 100644 --- a/playground/app.js +++ b/playground/app.js @@ -161,10 +161,7 @@ class MongoService { } } -const mongoservice = new MongoService(); -const mongoBackendPlugin = servicePluginFactory(mongoservice); - -class DataService { +class JsDataService { constructor() { this.dataDomains = { defaultDataset: [{ @@ -204,9 +201,17 @@ class DataService { }; } } + + async loadCSV(file) { + throw Error('Not implemented'); + } } -const dataService = new DataService(); +/* Use either: + - `JsDataService`: to execute all data transforms directly in the browser + - `MongoService`: to query against a Mongo database + */ +const dataService = new JsDataService(); const dataBackendPlugin = servicePluginFactory(dataService); async function buildVueApp() { @@ -258,7 +263,7 @@ async function buildVueApp() { groupname: 'Group 1', }, }); - const collections = await mongoservice.listCollections(); + const collections = await dataService.listCollections(); store.commit(VQBnamespace('setDomains'), { domains: collections }); store.dispatch(VQBnamespace('updateDataset')); }, @@ -325,7 +330,7 @@ async function buildVueApp() { this.draggedover = false; event.preventDefault(); // For the moment, only take one file and we should also test event.target - const { collection: domain } = await mongoservice.loadCSV(event.dataTransfer.files[0]); + const { collection: domain } = await dataService.loadCSV(event.dataTransfer.files[0]); await setupInitialData(store, domain); event.target.value = null; }, From 5a686873063f5c63a7122c0ecf8450e0c294a01e Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 11:08:21 +0200 Subject: [PATCH 8/9] chore: use featurepeek to deploy plaground Now that the playground can work client-side only, it's interesting to deploy it as a static website for easy reviews. --- .github/workflows/playground.yml | 35 ++++++++++++++++++++++++++++++++ .github/workflows/storybook.yml | 4 ++-- peek.yml | 6 +++++- 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/playground.yml diff --git a/.github/workflows/playground.yml b/.github/workflows/playground.yml new file mode 100644 index 0000000000..42b812ffc2 --- /dev/null +++ b/.github/workflows/playground.yml @@ -0,0 +1,35 @@ +# From https://docs.featurepeek.com/github-actions/ +name: Build playground +on: push +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: '12.x' + - name: Cache node modules + uses: actions/cache@v1 + with: + path: node_modules + key: ${{ runner.OS }}-build-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.OS }}-build-${{ env.cache-name }}- + ${{ runner.OS }}-build- + ${{ runner.OS }}- + - name: Install dependencies + run: yarn --frozen-lockfile + - name: Build Weaverbird module + run: yarn build-bundle + - name: Copy weaverbird script into dist folder + run: cp --remove-destination dist/weaverbird.browser.js playground/dist/weaverbird.browser.js + - name: Copy weaverbird CSS into dist folder + run: cp --remove-destination dist/weaverbird.css playground/dist/playground.css + - name: Copy playground script into dist folder + run: cp --remove-destination playground/app.js playground/dist/app.js + - name: Ping FeaturePeek + run: bash <(curl -s https://peek.run/ci) -a playground + env: + CI: true diff --git a/.github/workflows/storybook.yml b/.github/workflows/storybook.yml index ee4e2468a1..c24f68ae95 100644 --- a/.github/workflows/storybook.yml +++ b/.github/workflows/storybook.yml @@ -23,12 +23,12 @@ jobs: ${{ runner.OS }}-build- ${{ runner.OS }}- - name: Install dependencies - run: yarn + run: yarn --frozen-lockfile - name: Build Weaverbird components bundle run: yarn storybook:bundle - name: Build storybook run: yarn storybook:build - name: Ping FeaturePeek - run: bash <(curl -s https://peek.run/ci) + run: bash <(curl -s https://peek.run/ci) -a storybook env: CI: true diff --git a/peek.yml b/peek.yml index 3b8a2c90bf..3afd74d3f7 100644 --- a/peek.yml +++ b/peek.yml @@ -1,5 +1,9 @@ version: 2 -main: +playground: + type: static + path: /playground/dist + +storybook: type: static path: /.storybook/dist From a46d75f4ed359a28307988b14a0e734d91ae8cf3 Mon Sep 17 00:00:00 2001 From: David Nowinsky Date: Fri, 28 Aug 2020 11:56:56 +0200 Subject: [PATCH 9/9] fixup! feat: init js translator --- tests/unit/translator.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/translator.spec.ts b/tests/unit/translator.spec.ts index d73a3f6cc3..f8b623679a 100644 --- a/tests/unit/translator.spec.ts +++ b/tests/unit/translator.spec.ts @@ -72,7 +72,7 @@ describe('translator registration', () => { it('should be possible to register backends', () => { registerTranslator('dummy', DummyStringTranslator); expect(backendsSupporting('aggregate')).toEqual(['mongo36', 'mongo40']); - expect(backendsSupporting('domain')).toEqual(['dummy', 'mongo36', 'mongo40']); + expect(backendsSupporting('domain')).toEqual(['dummy', 'js', 'mongo36', 'mongo40']); }); it('should throw an error if backend is not available', () => { @@ -82,6 +82,6 @@ describe('translator registration', () => { it('should be possible to get all available translators', () => { registerTranslator('dummy', DummyStringTranslator); const translators = availableTranslators(); - expect(Object.keys(translators).sort()).toEqual(['dummy', 'mongo36', 'mongo40']); + expect(Object.keys(translators).sort()).toEqual(['dummy', 'js', 'mongo36', 'mongo40']); }); });