From 00d15bf4de2755908912c03d934d2641f9976551 Mon Sep 17 00:00:00 2001 From: Dieter Oberkofler Date: Thu, 19 Jun 2014 14:56:31 +0200 Subject: [PATCH] Added API to directly parse a json string containg an array --- CHANGELOG | 28 ++++--- install.sql | 14 ++-- json_array.tpb | 11 +++ json_array.tps | 1 + json_object.tpb | 2 +- json_parser.pkb | 140 ++++++++++++++++++--------------- json_parser.pks | 6 +- json_value.tpb | 13 +++ json_value.tps | 1 + unittest/json_ut.pkb | 178 ++++++++++++++++++++++++++++++++++++++++-- unittest/unittest.sql | 14 ++++ 11 files changed, 320 insertions(+), 88 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7ad051e..c6f1038 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,18 @@ -v0.1.0: - date: 2014-04-26 - changes: - - Added support for DATE types. - - Added support for JSONP. - -v0.0.1: - date: 2013-09-24 - changes: - - Initial release of plsql_json. +v0.2.0 - June 19, 2014 + +* 0.2.0 (Dieter Oberkofler) +* Now using 3 individual parse methods in json_parser allowing to parse an object, an array or any of the two. (Dieter Oberkofler) +* Added a new constructor to json_array allowing to parse a JSON string representing an array. Proposed by matthias-oe. (Dieter Oberkofler) +* Added a new constructor to json_value allowing to parse a JSON string representing an object or an array. (Dieter Oberkofler) +* Added unit tests for the new functionality. (Dieter Oberkofler) + +v0.1.0 - April 26, 2014 + +* 0.1.0 (Dieter Oberkofler) +* Added support for DATE types. +* Added support for JSONP. + +v0.0.1 - September 24, 2013 + +* 0.0.1 (Dieter Oberkofler) +* Initial release of plsql_json. diff --git a/install.sql b/install.sql index efbcdac..790315b 100644 --- a/install.sql +++ b/install.sql @@ -16,21 +16,15 @@ @@uninstall.sql --- install the types +-- install the headers @@json_keys.tps show errors @@json_node.tps show errors -@@json_node.tpb -show errors @@json_nodes.tps show errors @@json_value.tps show errors -@@json_value.tpb -show errors - --- install the packages @@json_object.tps show errors @@json_array.tps @@ -41,6 +35,12 @@ show errors show errors @@json_debug.pks show errors + +-- install the bodies +@@json_node.tpb +show errors +@@json_value.tpb +show errors @@json_object.tpb show errors @@json_array.tpb diff --git a/json_array.tpb b/json_array.tpb index ffbda4e..f45da8b 100644 --- a/json_array.tpb +++ b/json_array.tpb @@ -28,6 +28,17 @@ BEGIN RETURN; END json_array; +---------------------------------------------------------- +-- json_array +-- +CONSTRUCTOR FUNCTION json_array(SELF IN OUT NOCOPY json_array, theJSONString IN CLOB) RETURN SELF AS result +IS +BEGIN + SELF.nodes := json_parser.parse_array(theJSONString); + SELF.lastID := NULL; + RETURN; +END json_array; + ---------------------------------------------------------- -- append -- diff --git a/json_array.tps b/json_array.tps index ef64e09..4f3e501 100644 --- a/json_array.tps +++ b/json_array.tps @@ -7,6 +7,7 @@ TYPE json_array IS OBJECT -- Constructors CONSTRUCTOR FUNCTION json_array(self IN OUT NOCOPY json_array) RETURN self AS result, CONSTRUCTOR FUNCTION json_array(SELF IN OUT NOCOPY json_array, theData IN json_value) RETURN SELF AS result, + CONSTRUCTOR FUNCTION json_array(SELF IN OUT NOCOPY json_array, theJSONString IN CLOB) RETURN SELF AS result, -- Member setter methods MEMBER PROCEDURE append(self IN OUT NOCOPY json_array), diff --git a/json_object.tpb b/json_object.tpb index 22164b8..f6e8142 100644 --- a/json_object.tpb +++ b/json_object.tpb @@ -34,7 +34,7 @@ END json_object; CONSTRUCTOR FUNCTION json_object(SELF IN OUT NOCOPY json_object, theJSONString IN CLOB) RETURN SELF AS result IS BEGIN - SELF.nodes := json_parser.parser(theJSONString); + SELF.nodes := json_parser.parse_object(theJSONString); SELF.lastID := NULL; RETURN; END json_object; diff --git a/json_parser.pkb b/json_parser.pkb index 0c75399..2edbe01 100644 --- a/json_parser.pkb +++ b/json_parser.pkb @@ -32,6 +32,8 @@ FUNCTION lexer(jsrc IN OUT NOCOPY json_src) RETURN lTokens; PROCEDURE parseMem(tokens lTokens, indx IN OUT PLS_INTEGER, mem_name VARCHAR2, mem_indx NUMBER, theParentID IN OUT BINARY_INTEGER, theLastID IN OUT BINARY_INTEGER, theNodes IN OUT NOCOPY json_nodes); +FUNCTION parse(tokens IN lTokens, firstToken IN VARCHAR2) RETURN json_nodes; + ---------------------------------------------------------- -- GLOBAL MODULES ---------------------------------------------------------- @@ -630,7 +632,7 @@ BEGIN RETURN; ELSE - p_error('Expected string or }', tok); + p_error('Expected string or } but found '||tok.type_name, tok); END CASE; @@ -788,7 +790,7 @@ BEGIN p_error('Premature exit in array', tok); END IF; ELSIF (tok.type_name != ']') THEN --error - p_error('Expected , or ]', tok); + p_error('Expected , or ] but found '||tok.type_name, tok); END IF; END LOOP; @@ -929,52 +931,17 @@ BEGIN END parseMem; ---------------------------------------------------------- --- parse_list +-- parse -- -FUNCTION parse_list(str CLOB) RETURN json_nodes +FUNCTION parse(tokens IN lTokens, firstToken IN VARCHAR2) RETURN json_nodes IS - tokens lTokens; - --yyy obj json_list; - obj json_nodes := json_nodes(); - indx PLS_INTEGER := 1; - jsrc json_src; -BEGIN - debug('parse_list'); - updateDecimalPoint(); - jsrc := prepareClob(str); - tokens := lexer(jsrc); - IF (tokens(indx).type_name = '[') THEN - indx := indx + 1; - --yyy obj := parseArr(tokens, indx); - ELSE - raise_application_error(-20101, 'JSON List Parser exception - no [ start found'); - END IF; - IF (tokens.count != indx) THEN - p_error('] should end the JSON List object', tokens(indx)); - END IF; - - RETURN obj; -END parse_list; - ----------------------------------------------------------- --- parser --- -FUNCTION parser(str CLOB) RETURN json_nodes -IS - tokens lTokens; - obj json_nodes := json_nodes(); - - indx PLS_INTEGER := 1; - jsrc json_src; - i BINARY_INTEGER; + lastToken VARCHAR2(1) := NULL; + nodes json_nodes := json_nodes(); + indx PLS_INTEGER := 1; + --i BINARY_INTEGER := NULL; aParentID BINARY_INTEGER := NULL; aLastID BINARY_INTEGER := NULL; BEGIN - updateDecimalPoint(); - jsrc := prepareClob(str); - - tokens := lexer(jsrc); - -- dump tokens /* dbms_output.put_line('----------LEXER-S----------'); @@ -986,42 +953,93 @@ BEGIN dbms_output.put_line('----------LEXER-E----------'); */ + IF (tokens(indx).type_name != firstToken) THEN + raise_application_error(-20101, 'JSON Parser exception - invalid first token. Expected:'||firstToken||' bit found:'||tokens(indx).type_name); + END IF; + IF (tokens(indx).type_name = '{') THEN + lastToken := '}'; + indx := indx + 1; + parseObj(tokens, indx, aParentID, aLastID, nodes); + ELSIF (tokens(indx).type_name = '[') THEN + lastToken := ']'; indx := indx + 1; - --yyy obj := parseObj(tokens, indx); - parseObj(tokens, indx, aParentID, aLastID, obj); + parseArr(tokens, indx, aParentID, aLastID, nodes); ELSE - raise_application_error(-20101, 'JSON Parser exception - no { start found'); + raise_application_error(-20101, 'JSON Parser exception - no '||firstToken||' start found'); END IF; IF (tokens.count != indx) THEN - p_error('} should end the JSON object', tokens(indx)); + p_error(lastToken||' should end the last token in the JSON string', tokens(indx)); END IF; - RETURN obj; -END parser; + RETURN nodes; +END parse; ---------------------------------------------------------- --- parse_any +-- parse_object -- -FUNCTION parse_any(str CLOB) RETURN /*yyy json_value*/json_nodes +FUNCTION parse_object(str CLOB) RETURN json_nodes IS + jsrc json_src; tokens lTokens; - --yyy obj json_list; - obj json_array := json_array(); - indx PLS_INTEGER := 1; +BEGIN + updateDecimalPoint(); + jsrc := prepareClob(str); + tokens := lexer(jsrc); + + IF (tokens(1).type_name != '{') THEN + raise_application_error(-20101, 'JSON Parser exception - invalid first token = '||tokens(1).type_name); + END IF; + + RETURN parse(tokens=>tokens, firstToken=>'{'); +END parse_object; + +---------------------------------------------------------- +-- parse_array +-- +FUNCTION parse_array(str CLOB) RETURN json_nodes +IS jsrc json_src; + tokens lTokens; BEGIN - debug('parse_any'); + updateDecimalPoint(); jsrc := prepareClob(str); tokens := lexer(jsrc); - tokens(tokens.count+1).type_name := ']'; - --yyy obj := parseArr(tokens, indx); - IF (tokens.count != indx) THEN - p_error('] should end the JSON List object', tokens(indx)); + + IF (tokens(1).type_name != '[') THEN + raise_application_error(-20101, 'JSON Parser exception - invalid first token = '||tokens(1).type_name); + END IF; + + RETURN parse(tokens=>tokens, firstToken=>'['); +END parse_array; + +---------------------------------------------------------- +-- parse_any +-- +FUNCTION parse_any(str CLOB) RETURN json_value +IS + firstToken VARCHAR2(1); + jsrc json_src; + tokens lTokens; + value json_value := json_value(); +BEGIN + updateDecimalPoint(); + jsrc := prepareClob(str); + tokens := lexer(jsrc); + + IF (tokens(1).type_name = '{') THEN + firstToken := tokens(1).type_name; + value.typ := 'O'; + ELSIF (tokens(1).type_name = '[') THEN + firstToken := tokens(1).type_name; + value.typ := 'A'; + ELSE + raise_application_error(-20101, 'JSON Parser exception - invalid first token = '||tokens(1).type_name); END IF; - --yyy return obj.head(); - RETURN NULL; + value.nodes := parse(tokens=>tokens, firstToken=>firstToken); + + RETURN value; END parse_any; END json_parser; diff --git a/json_parser.pks b/json_parser.pks index a8a0e25..834bf6a 100644 --- a/json_parser.pks +++ b/json_parser.pks @@ -49,9 +49,9 @@ json_strict BOOLEAN NOT NULL := FALSE; -- GLOBAL PUBLIC MODULES ---------------------------------------------------------- -FUNCTION parser(str CLOB) RETURN json_nodes; -FUNCTION parse_list(str CLOB) RETURN json_nodes; -FUNCTION parse_any(str CLOB) RETURN json_nodes; +FUNCTION parse_object(str CLOB) RETURN json_nodes; +FUNCTION parse_array(str CLOB) RETURN json_nodes; +FUNCTION parse_any(str CLOB) RETURN json_value; END json_parser; / diff --git a/json_value.tpb b/json_value.tpb index 38c9918..e0e0591 100644 --- a/json_value.tpb +++ b/json_value.tpb @@ -14,6 +14,19 @@ BEGIN RETURN; END json_value; +---------------------------------------------------------- +-- json_value +-- +CONSTRUCTOR FUNCTION json_value(SELF IN OUT NOCOPY json_value, theJSONString IN CLOB) RETURN SELF AS RESULT +IS + value json_value := json_value(); +BEGIN + value := json_parser.parse_any(theJSONString); + SELF.typ := value.typ; + SELF.nodes := value.nodes; + RETURN; +END json_value; + ---------------------------------------------------------- -- get_type -- diff --git a/json_value.tps b/json_value.tps index 912ad72..f70701a 100644 --- a/json_value.tps +++ b/json_value.tps @@ -6,6 +6,7 @@ TYPE json_value IS OBJECT -- Default constructor CONSTRUCTOR FUNCTION json_value(SELF IN OUT NOCOPY json_value) RETURN SELF AS RESULT, + CONSTRUCTOR FUNCTION json_value(SELF IN OUT NOCOPY json_value, theJSONString IN CLOB) RETURN SELF AS RESULT, -- Member getter methods MEMBER FUNCTION get_type RETURN VARCHAR2, diff --git a/unittest/json_ut.pkb b/unittest/json_ut.pkb index bdd6d68..1572db2 100644 --- a/unittest/json_ut.pkb +++ b/unittest/json_ut.pkb @@ -665,9 +665,9 @@ BEGIN END UT_BigObject; ---------------------------------------------------------- --- UT_ParseBasic (private) +-- UT_ParseObject (private) -- -PROCEDURE UT_ParseBasic +PROCEDURE UT_ParseObject IS TYPE TestValueType IS RECORD (v VARCHAR2(32767), r VARCHAR2(32767)); TYPE TestValueList IS TABLE OF TestValueType INDEX BY BINARY_INTEGER; @@ -686,12 +686,13 @@ IS END addPair; BEGIN - UT_util.module('UT_ParseBasic'); + UT_util.module('UT_ParseObject'); -- allocate clob dbms_lob.createtemporary(aLob, TRUE); - addPair('{ }'); + addPair('{}'); + addPair(' { } '); addPair('{"p1": null}'); addPair('{"p1": "v1"}'); addPair('{"p1": -0.4711}'); @@ -723,7 +724,170 @@ BEGIN -- free temporary CLOB dbms_lob.freetemporary(aLob); -END UT_ParseBasic; +END UT_ParseObject; + +---------------------------------------------------------- +-- UT_ParseArray (private) +-- +PROCEDURE UT_ParseArray +IS + TYPE TestValueType IS RECORD (v VARCHAR2(32767), r VARCHAR2(32767)); + TYPE TestValueList IS TABLE OF TestValueType INDEX BY BINARY_INTEGER; + + aList TestValueList; + aArray json_array := json_array(); + aLob CLOB := empty_clob(); + i BINARY_INTEGER; + + PROCEDURE addPair(theValue IN VARCHAR2, theResult IN VARCHAR2 DEFAULT NULL) + IS + c BINARY_INTEGER := aList.COUNT + 1; + BEGIN + aList(c).v := theValue; + aList(c).r := NVL(theResult, REPLACE(theValue, ' ', '')); + END addPair; + +BEGIN + UT_util.module('UT_ParseArray'); + + -- allocate clob + dbms_lob.createtemporary(aLob, TRUE); + + addPair('[]'); + addPair(' [ ] '); + addPair('[null]'); + addPair('["v1"]'); + addPair('[-0.4711]'); + addPair('[true]'); + addPair('[false]'); + addPair('["v1", 2, true]'); + addPair('[[{"a1p1": "a1v1", "a1p2": {}}, {"a2p1": "a2v2", "a2p2": []}, {"a3p1": "a3v1", "a3p2": [1, 2, 3]}]]'); + addPair('[{}]'); + addPair('[[]]'); + addPair('[[{}, {}, {}, [[], {}, [[]]]]]'); + + FOR i IN 1 .. aList.COUNT LOOP + -- parse the json string + aArray := json_array(aList(i).v); + + -- validate the resulting object + json_utils.validate(aArray.nodes); + + -- convert the object back to a version string + aArray.to_clob(aLob); + + -- test + UT_util.eqLOB( theTitle => '#'||i||': '||aList(i).v, + theComputed => aLob, + theExpected => aList(i).r, + theNullOK => TRUE + ); + END LOOP; + + -- free temporary CLOB + dbms_lob.freetemporary(aLob); +END UT_ParseArray; + +---------------------------------------------------------- +-- UT_ParseAny (private) +-- +PROCEDURE UT_ParseAny +IS + TYPE TestValueType IS RECORD (v VARCHAR2(32767), r VARCHAR2(32767)); + TYPE TestValueList IS TABLE OF TestValueType INDEX BY BINARY_INTEGER; + + aList TestValueList; + + aValue json_value := json_value(); + aStrBuf VARCHAR2(32767); + aLob CLOB := empty_clob(); + + i PLS_INTEGER; + + PROCEDURE addPair(theValue IN VARCHAR2, theResult IN VARCHAR2 DEFAULT NULL) + IS + c BINARY_INTEGER := aList.COUNT + 1; + BEGIN + aList(c).v := theValue; + aList(c).r := NVL(theResult, REPLACE(theValue, ' ', '')); + END addPair; + +BEGIN + UT_util.module('UT_ParseAny'); + + -- allocate clob + dbms_lob.createtemporary(aLob, TRUE); + + addPair('{}'); + addPair(' { } '); + addPair('{"p1": null}'); + addPair('{"p1": "v1"}'); + addPair('{"p1": -0.4711}'); + addPair('{"p1": true}'); + addPair('{"p1": false}'); + addPair('{"p1": "v1", "p2": 2, "p3": true}'); + addPair('{"p1": [{"a1p1": "a1v1", "a1p2": {}}, {"a2p1": "a2v2", "a2p2": []}, {"a3p1": "a3v1", "a3p2": [1, 2, 3]}]}'); + addPair('{"p1": {}}'); + addPair('{"p1": []}'); + addPair('{"p1": [{}, {}, {}, [[], {}, [[]]]]}'); + addPair('[]'); + addPair(' [ ] '); + addPair('[null]'); + addPair('["v1"]'); + addPair('[-0.4711]'); + addPair('[true]'); + addPair('[false]'); + addPair('["v1", 2, true]'); + addPair('[[{"a1p1": "a1v1", "a1p2": {}}, {"a2p1": "a2v2", "a2p2": []}, {"a3p1": "a3v1", "a3p2": [1, 2, 3]}]]'); + addPair('[{}]'); + addPair('[[]]'); + addPair('[[{}, {}, {}, [[], {}, [[]]]]]'); + + FOR i IN 1 .. aList.COUNT LOOP + -- use parses + aValue := json_parser.parse_any(aList(i).v); + json_utils.validate(aValue.nodes); + + json_utils.erase_clob(aLob); + aStrBuf := NULL; + IF (aValue.typ = 'O') THEN + json_utils.object_to_clob(theLobBuf=>aLob, theStrBuf=>aStrBuf, theNodes=>aValue.nodes, theNodeID=>aValue.nodes.FIRST); + ELSIF (aValue.typ = 'A') THEN + json_utils.array_to_clob(theLobBuf=>aLob, theStrBuf=>aStrBuf, theNodes=>aValue.nodes, theNodeID=>aValue.nodes.FIRST); + ELSE + NULL; + END IF; + + UT_util.eqLOB( theTitle => '#'||i||': '||aList(i).v, + theComputed => aLob, + theExpected => aList(i).r, + theNullOK => TRUE + ); + + -- use json_value to parse + aValue := json_value(aList(i).v); + json_utils.validate(aValue.nodes); + + json_utils.erase_clob(aLob); + aStrBuf := NULL; + IF (aValue.typ = 'O') THEN + json_utils.object_to_clob(theLobBuf=>aLob, theStrBuf=>aStrBuf, theNodes=>aValue.nodes, theNodeID=>aValue.nodes.FIRST); + ELSIF (aValue.typ = 'A') THEN + json_utils.array_to_clob(theLobBuf=>aLob, theStrBuf=>aStrBuf, theNodes=>aValue.nodes, theNodeID=>aValue.nodes.FIRST); + ELSE + NULL; + END IF; + + UT_util.eqLOB( theTitle => '#'||i||': '||aList(i).v, + theComputed => aLob, + theExpected => aList(i).r, + theNullOK => TRUE + ); + END LOOP; + + -- free temporary CLOB + dbms_lob.freetemporary(aLob); +END UT_ParseAny; ---------------------------------------------------------- -- UT_ParseSimple (private) @@ -1026,7 +1190,9 @@ BEGIN UT_DeepRecursion; UT_ComplexObject; UT_BigObject; - UT_ParseBasic; + UT_ParseObject; + UT_ParseArray; + UT_ParseAny; UT_ParseSimple; UT_ParseComplex; UT_ParseAndDestruct; diff --git a/unittest/unittest.sql b/unittest/unittest.sql index bdd6de7..ea6ebcc 100644 --- a/unittest/unittest.sql +++ b/unittest/unittest.sql @@ -12,6 +12,9 @@ */ +-- environment +set pagesize 10000 linesize 10000 trimout on trimspool on + -- delete existing objects whenever sqlerror continue DROP TABLE UT_test_table; @@ -57,6 +60,17 @@ UNION SELECT 'Failed unit tests: '||C "Unit test results" FROM (SELECT COUNT(*) C FROM plsql_json.UT_test_table WHERE Success = 'N') ORDER BY 1 DESC; +-- show the errors +column Module format a30 +column Title format a30 +column Result format a30 +column Expected format a30 +column Computed format a30 +SELECT ID, Module, Title, Result, Expected, Computed +FROM UT_test_table +WHERE Success = 'N' +ORDER BY ID; + -- cleanup whenever sqlerror continue DROP TABLE UT_test_table;