Skip to content

Commit

Permalink
Build end-to-end prototype with basic functionalities.
Browse files Browse the repository at this point in the history
  • Loading branch information
xwkuang5 committed Dec 1, 2021
1 parent 7ef52a3 commit f63abbe
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 4 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "calcite"]
path = calcite
url = https://github.com/apache/calcite.git
1 change: 1 addition & 0 deletions calcite
Submodule calcite added at 065e32
48 changes: 48 additions & 0 deletions ledger_sql.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/bin/bash

if [[ "$#" -ne 2 ]]
then
echo "Illegal number of parameters."
echo "Usage: /path/to/ledger_sql.sh </path/to/ledger> </path/to/scratch/dir>"
exit
fi

LEDGER_FILE="$1"
SCRATCH_DIR="$2"
CALCITE_LEDGER_SCHEMA_NAME=LEDGER
CALCITE_LEDGER_SCHEMA_DIR="$SCRATCH_DIR/LEDGER"

clean_up () {
ARG=$?
rm -rf $SCRATCH_DIR
exit $ARG
}
trap clean_up EXIT

# Initialize Calcite.
git submodule update --init --recursive

ROOT_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"

pushd $ROOT_DIR/calcite && ./gradlew -q :example:csv:buildSqllineClasspath && popd >/dev/null 2>&1

mkdir -p $SCRATCH_DIR

# Populate model.json from template
sed "s/<calcite_ledger_schema_name>/$CALCITE_LEDGER_SCHEMA_NAME/g" $ROOT_DIR/model.json > "$SCRATCH_DIR/model.json"

# Use # as separator in sed instead of / to avoid confusion with / in the path.
sed -i "s#<calcite_ledger_schema_dir>#$CALCITE_LEDGER_SCHEMA_DIR#g" "$SCRATCH_DIR/model.json"

mkdir -p $CALCITE_LEDGER_SCHEMA_DIR

# Generate the CSV output of the ledger file.
$ROOT_DIR/txn_gen.sh $LEDGER_FILE $CALCITE_LEDGER_SCHEMA_DIR/LEDGER.csv "$SCRATCH_DIR/scratchfile"

# Start sqlline.
java \
-Djavax.xml.parsers.DocumentBuilderFactory=com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl \
-jar $ROOT_DIR/calcite/example/csv/build/libs/sqllineClasspath.jar \
-u jdbc:calcite:model=$SCRATCH_DIR/model.json \
-n admin \
-p admin
14 changes: 14 additions & 0 deletions model.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"version": "1.0",
"defaultSchema": "<calcite_ledger_schema_name>",
"schemas": [
{
"name": "<calcite_ledger_schema_name>",
"type": "custom",
"factory": "org.apache.calcite.adapter.csv.CsvSchemaFactory",
"operand": {
"directory": "<calcite_ledger_schema_dir>"
}
}
]
}
8 changes: 4 additions & 4 deletions txn_gen.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
# TODO(louiskuang): consider switching to a python script and add unit tests.
# TODO(louiskuang): support the notes column %(quoted(join(note | xact.note))).
# TODO(louiskuang): make filename part of txn id.
# TODO(xwkuang5): consider switching to a python script and add unit tests.
# TODO(xwkuang5): support the notes column %(quoted(join(note | xact.note))).
# TODO(xwkuang5): make filename part of txn id.

function join_by { local d=${1-} f=${2-}; if shift 2; then printf %s "$f" "${@/#/$d}"; fi; }

Expand Down Expand Up @@ -30,7 +30,7 @@ NUM_USER_VISIBLE_FIELDS=7
USER_VISIBLE_FIELD_START_IDX=2

# Calcite header.
HEADER="TXN_ID:long,TXN_DATE:date,CODE:string,PAYEE:string,ACCOUNT:string,COMMODITY:string,QUANTITY:numeric,STATUS:string"
HEADER="TXN_ID:long,TXN_DATE:date,CODE:string,PAYEE:string,ACCOUNT:string,COMMODITY:string,QUANTITY:\"decimal(18,2)\",STATUS:string"

# Truncate the output.
: > $2
Expand Down

0 comments on commit f63abbe

Please sign in to comment.