diff --git a/eggstrain/Cargo.toml b/eggstrain/Cargo.toml index cab8161..b717bb0 100644 --- a/eggstrain/Cargo.toml +++ b/eggstrain/Cargo.toml @@ -14,9 +14,8 @@ authors = [ anyhow = "1" arrow = "50" async-trait = "0.1" -# datafusion = "35" +datafusion = "35" serde_json = "1" -substrait = { version = "0.24", features = ["pbjson"] } tokio = { version = "1", features = ["full"] } tokio-stream = "0.1" rayon = "1" diff --git a/substrait/basic_query.json b/substrait/basic_query.json deleted file mode 100644 index ed1b787..0000000 --- a/substrait/basic_query.json +++ /dev/null @@ -1,393 +0,0 @@ -{ - "extensions": [ - { - "extensionFunction": { - "functionAnchor": 1, - "name": "equal" - } - }, - { - "extensionFunction": { - "functionAnchor": 2, - "name": "is_not_null" - } - }, - { - "extensionFunction": { - "functionAnchor": 3, - "name": "and" - } - }, - { - "extensionFunction": { - "functionAnchor": 4, - "name": "lt" - } - } - ], - "relations": [ - { - "root": { - "input": { - "project": { - "input": { - "project": { - "input": { - "join": { - "left": { - "read": { - "baseSchema": { - "names": [ - "author_id", - "name", - "nationality" - ], - "struct": { - "types": [ - { - "i32": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "varchar": { - "length": 19, - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "varchar": { - "length": 7, - "nullability": "NULLABILITY_NULLABLE" - } - } - ], - "nullability": "NULLABILITY_REQUIRED" - } - }, - "filter": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "varchar": { - "length": 1, - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - } - }, - { - "value": { - "literal": { - "string": "England" - } - } - } - ] - } - } - }, - { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "varchar": { - "length": 1, - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - } - } - ] - } - } - } - ] - } - }, - "projection": { - "select": { - "structItems": [ - {}, - { - "field": 1 - } - ] - }, - "maintainSingularStruct": true - }, - "namedTable": { - "names": ["Authors"] - } - } - }, - "right": { - "read": { - "baseSchema": { - "names": [ - "book_id", - "title", - "author_id", - "publication_year" - ], - "struct": { - "types": [ - { - "i32": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "varchar": { - "length": 20, - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "i32": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - } - ], - "nullability": "NULLABILITY_REQUIRED" - } - }, - "filter": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "scalarFunction": { - "functionReference": 4, - "outputType": { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 3 - } - }, - "rootReference": {} - } - } - }, - { - "value": { - "literal": { - "i32": 1850 - } - } - } - ] - } - } - }, - { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 3 - } - }, - "rootReference": {} - } - } - } - ] - } - } - } - ] - } - }, - "projection": { - "select": { - "structItems": [ - { - "field": 2 - }, - { - "field": 1 - } - ] - }, - "maintainSingularStruct": true - }, - "namedTable": { - "names": ["Books"] - } - } - }, - "expression": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": {} - }, - "rootReference": {} - } - } - }, - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - } - } - ] - } - }, - "type": "JOIN_TYPE_INNER" - } - }, - "expressions": [ - { - "selection": { - "directReference": { - "structField": {} - }, - "rootReference": {} - } - }, - { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - }, - "rootReference": {} - } - }, - { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - }, - { - "selection": { - "directReference": { - "structField": { - "field": 3 - } - }, - "rootReference": {} - } - } - ] - } - }, - "expressions": [ - { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - }, - "rootReference": {} - } - }, - { - "selection": { - "directReference": { - "structField": { - "field": 3 - } - }, - "rootReference": {} - } - } - ] - } - }, - "names": ["author_name", "book_title"] - } - } - ], - "version": { - "minorNumber": 24, - "producer": "DuckDB" - } -} diff --git a/substrait/examples.md b/substrait/examples.md deleted file mode 100644 index e4639b3..0000000 --- a/substrait/examples.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# basic_query - -``` -CREATE TABLE Authors ( - author_id INT PRIMARY KEY, - name VARCHAR(255) NOT NULL, - nationality VARCHAR(50) -); - -CREATE TABLE Books ( - book_id INT PRIMARY KEY, - title VARCHAR(255) NOT NULL, - author_id INT NOT NULL, - publication_year INT DEFAULT 0, - FOREIGN KEY (author_id) REFERENCES Authors(author_id) -); - -INSERT INTO Authors (author_id, name, nationality) VALUES -(1, 'Jane Austen', 'England'), -(2, 'William Shakespeare', 'England'), -(3, 'Victor Hugo', 'France'), -(4, 'Fyodor Dostoevsky', 'Russia'); - -INSERT INTO Books (book_id, title, author_id, publication_year) VALUES -(1, 'Pride and Prejudice', 1, 1813), -(2, 'Hamlet', 2, 1603), -(3, 'Les Misérables', 3, 1862), -(4, 'Crime and Punishment', 4, 1866); - -SELECT - a.name AS author_name, - b.title AS book_title -FROM Authors a -JOIN Books b ON a.author_id = b.author_id -WHERE a.nationality = 'England' AND b.publication_year < 1850; -``` - - -# TPC-H Query 1 - -lineitem -``` -CREATE TABLE lineitem ( - l_orderkey BIGINT NOT NULL, - l_partkey BIGINT NOT NULL, - l_suppkey BIGINT NOT NULL, - l_linenumber INTEGER NOT NULL, - l_quantity INTEGER NOT NULL, - l_shipdate DATE NOT NULL, - l_discount DOUBLE PRECISION NOT NULL, - l_tax DOUBLE PRECISION NOT NULL, - l_returnflag CHAR(1) NOT NULL, - l_linestatus CHAR(2) NOT NULL, - l_shipinstruct CHAR(25) NOT NULL, - l_shipmode CHAR(10) NOT NULL, - l_comment VARCHAR(44) NOT NULL, - PRIMARY KEY (l_orderkey, l_linenumber) -); -``` - -# orders -``` -CREATE TABLE orders ( -o_orderkey BIGINT NOT NULL PRIMARY KEY, -o_custkey BIGINT NOT NULL, -o_orderstatus CHAR(1) NOT NULL, -o_totalprice DECIMAL NOT NULL, -o_orderdate DATE NOT NULL, -o_orderpriority CHAR(15) NOT NULL, -o_clerk CHAR(15) NOT NULL, -o_shippriority INTEGER NOT NULL, -o_comment VARCHAR(79) NOT NULL, -); -``` \ No newline at end of file diff --git a/substrait/read_rel_example.json b/substrait/read_rel_example.json deleted file mode 100644 index 8481958..0000000 --- a/substrait/read_rel_example.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "baseSchema": { - "names": ["author_id", "name", "nationality"], - "struct": { - "types": [ - { - "i32": { - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "varchar": { - "length": 19, - "nullability": "NULLABILITY_REQUIRED" - } - }, - { - "varchar": { - "length": 7, - "nullability": "NULLABILITY_NULLABLE" - } - } - ], - "nullability": "NULLABILITY_REQUIRED" - } - }, - "filter": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "varchar": { - "length": 1, - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - } - }, - { - "value": { - "literal": { - "string": "England" - } - } - } - ] - } - } - }, - { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "varchar": { - "length": 1, - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 2 - } - }, - "rootReference": {} - } - } - } - ] - } - } - } - ] - } - }, - "projection": { - "select": { - "structItems": [ - {}, - { - "field": 1 - } - ] - }, - "maintainSingularStruct": true - }, - "namedTable": { - "names": ["Authors"] - } -} diff --git a/substrait/substrait_plan_example.json b/substrait/substrait_plan_example.json deleted file mode 100644 index 54095be..0000000 --- a/substrait/substrait_plan_example.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "extensions": [ - { - "extensionFunction": { - "functionAnchor": 1, - "name": "lte" - } - }, - { - "extensionFunction": { - "functionAnchor": 2, - "name": "is_not_null" - } - }, - { - "extensionFunction": { - "functionAnchor": 3, - "name": "and" - } - }, - { - "extensionFunction": { - "functionAnchor": 4, - "name": "count" - } - } - ], - "relations": [ - { - "root": { - "input": { - "project": { - "input": { - "aggregate": { - "input": { - "read": { - "baseSchema": { - "names": [ - "exercise", - "difficulty_level" - ], - "struct": { - "types": [ - { - "varchar": { - "length": 13, - "nullability": "NULLABILITY_NULLABLE" - } - }, - { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - } - ], - "nullability": "NULLABILITY_REQUIRED" - } - }, - "filter": { - "scalarFunction": { - "functionReference": 3, - "outputType": { - "bool": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "scalarFunction": { - "functionReference": 1, - "outputType": { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - }, - "rootReference": {} - } - } - }, - { - "value": { - "literal": { - "i32": 5 - } - } - } - ] - } - } - }, - { - "value": { - "scalarFunction": { - "functionReference": 2, - "outputType": { - "i32": { - "nullability": "NULLABILITY_NULLABLE" - } - }, - "arguments": [ - { - "value": { - "selection": { - "directReference": { - "structField": { - "field": 1 - } - }, - "rootReference": {} - } - } - } - ] - } - } - } - ] - } - }, - "projection": { - "select": { - "structItems": [{}] - }, - "maintainSingularStruct": true - }, - "namedTable": { - "names": ["crossfit"] - } - } - }, - "groupings": [{}], - "measures": [ - { - "measure": { - "functionReference": 4, - "outputType": { - "i64": { - "nullability": "NULLABILITY_NULLABLE" - } - } - } - } - ] - } - }, - "expressions": [ - { - "selection": { - "directReference": { - "structField": {} - }, - "rootReference": {} - } - } - ] - } - }, - "names": ["exercise"] - } - } - ], - "version": { - "minorNumber": 24, - "producer": "DuckDB" - } -}