Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow property graph creation on attached databases #171

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
aa62f1f
Adding in catalog information
Dtenwolde Nov 4, 2024
0a8bc95
Merge branch 'main' into issue/160-support-property-graphs-on-attache…
Dtenwolde Nov 7, 2024
266fd69
Add bluesky dataset
Dtenwolde Nov 7, 2024
49891c0
Update test, get correct catalog, updating match
Dtenwolde Nov 8, 2024
4938d3b
Update refs
Dtenwolde Nov 26, 2024
2201d0b
Change alias_map to use pointer to PropertyGraphTable
Dtenwolde Nov 26, 2024
c004071
Remove toLower, add shared_ptr to PropertyGraphTable in edge table
Dtenwolde Nov 27, 2024
0b631cc
Now using that shared_ptr to propertygraphtable
Dtenwolde Nov 27, 2024
e93c47f
Rewrite to use pg_table instead of string
Dtenwolde Nov 27, 2024
528a74a
replace some std::string
Dtenwolde Nov 27, 2024
86687b4
Rework CreateJoin function and add optional argument for alias to cre…
Dtenwolde Nov 27, 2024
6ded840
Rework some base table ref
Dtenwolde Nov 27, 2024
7a42c84
Rework some more basetable ref
Dtenwolde Nov 27, 2024
73f0949
rework more base tableref
Dtenwolde Nov 27, 2024
47ff72a
Remove print, update tests
Dtenwolde Nov 27, 2024
4911960
Fix missing alias for edgetypeany
Dtenwolde Nov 27, 2024
93a6296
Fix bug, now using table name instead of label
Dtenwolde Nov 27, 2024
a7cb003
Remove print
Dtenwolde Nov 27, 2024
1a5621f
Find table by table name now, updated error messages
Dtenwolde Nov 27, 2024
1262a30
Merge branch 'main' into issue/160-support-property-graphs-on-attache…
Dtenwolde Nov 27, 2024
de82de8
Remove unused variable
Dtenwolde Nov 27, 2024
8d5a8d2
Fix build warning
Dtenwolde Nov 27, 2024
12dee72
Add lcc test
Dtenwolde Nov 27, 2024
d00a2df
Add tests for wcc and pagerank
Dtenwolde Nov 27, 2024
16e9313
Update test with path finding
Dtenwolde Nov 27, 2024
eae7ffb
Move bluesky database file
Dtenwolde Nov 27, 2024
20d0da9
Attaching to remote database works
Dtenwolde Nov 27, 2024
0927910
Change ref
Dtenwolde Nov 27, 2024
d9d7b25
Remove httpfs extension loading
Dtenwolde Nov 27, 2024
a40dace
Add some more tests and use the schema specified in the pg
Dtenwolde Nov 27, 2024
d38a8c0
Get the catalog and schema when opening new connection
Dtenwolde Nov 28, 2024
4886c6a
Also set catalog and schema. Refactor code to retrieve property graphs
Dtenwolde Nov 28, 2024
38cc21f
More tests
Dtenwolde Nov 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/core/functions/scalar/shortest_path.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,6 @@ static void ShortestPathFunction(DataChunk &args, ExpressionState &state,
throw ConstraintException(
"Need to initialize CSR before doing shortest path");
}

int32_t id = args.data[0].GetValue(0).GetValue<int32_t>();
int64_t v_size = args.data[1].GetValue(0).GetValue<int64_t>();

auto *v = (int64_t *)csr->v;
Expand Down
24 changes: 16 additions & 8 deletions src/core/functions/table/create_property_graph.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -169,21 +169,22 @@ unique_ptr<FunctionData> CreatePropertyGraphFunction::CreatePropertyGraphBind(
" already exists");
}

auto &catalog = Catalog::GetCatalog(context, info->catalog);
case_insensitive_set_t v_table_names;
for (auto &vertex_table : info->vertex_tables) {
try {
auto &catalog = Catalog::GetCatalog(context, vertex_table->catalog_name);
auto table = catalog.GetEntry<TableCatalogEntry>(
context, info->schema, vertex_table->table_name, OnEntryNotFound::RETURN_NULL);

if (!table) {
throw Exception(ExceptionType::INVALID,
"Table " + vertex_table->table_name + " not found");
"Table " + vertex_table->schema_name + "." + vertex_table->table_name + " not found");
}

CheckPropertyGraphTableColumns(vertex_table, *table);
CheckPropertyGraphTableLabels(vertex_table, *table);
} catch (CatalogException &e) {
auto &catalog = Catalog::GetCatalog(context, vertex_table->catalog_name);
auto table = catalog.GetEntry<ViewCatalogEntry>(context, info->schema, vertex_table->table_name, OnEntryNotFound::RETURN_NULL);
if (table) {
throw Exception(ExceptionType::INVALID, "Found a view with name " + vertex_table->table_name + ". Creating property graph tables over views is currently not supported.");
Expand All @@ -200,11 +201,13 @@ unique_ptr<FunctionData> CreatePropertyGraphFunction::CreatePropertyGraphBind(

for (auto &edge_table : info->edge_tables) {
try {
auto table = catalog.GetEntry<TableCatalogEntry>(context, info->schema,
auto &catalog = Catalog::GetCatalog(context, edge_table->catalog_name);

auto table = catalog.GetEntry<TableCatalogEntry>(context, edge_table->schema_name,
edge_table->table_name, OnEntryNotFound::RETURN_NULL);
if (!table) {
throw Exception(ExceptionType::INVALID,
"Table " + edge_table->table_name + " not found");
"Table " + edge_table->schema_name + "." + edge_table->table_name + " not found");
}

CheckPropertyGraphTableColumns(edge_table, *table);
Expand Down Expand Up @@ -237,6 +240,7 @@ unique_ptr<FunctionData> CreatePropertyGraphFunction::CreatePropertyGraphBind(
// Validate primary keys in the destination table
ValidatePrimaryKeyInTable(catalog, context, info->schema, edge_table->destination_reference, edge_table->destination_pk);
} catch (CatalogException &e) {
auto &catalog = Catalog::GetCatalog(context, edge_table->catalog_name);
auto table = catalog.GetEntry<ViewCatalogEntry>(context, info->schema, edge_table->table_name, OnEntryNotFound::RETURN_NULL);
if (table) {
throw Exception(ExceptionType::INVALID, "Found a view with name " + edge_table->table_name + ". Creating property graph tables over views is currently not supported.");
Expand Down Expand Up @@ -307,10 +311,12 @@ void CreatePropertyGraphFunction::CreatePropertyGraphFunc(
for (idx_t i = 0; i < v_table->sub_labels.size(); i++) {
insert_info += "'" + v_table->sub_labels[i] + (i == v_table->sub_labels.size() - 1 ? "'" : "', ");
}
insert_info += "]";
insert_info += "],";
} else {
insert_info += "NULL";
insert_info += "NULL,";
}
insert_info += "'" + v_table->catalog_name + "', ";
insert_info += "'" + v_table->schema_name + "'";
insert_info += "), ";
}

Expand Down Expand Up @@ -350,10 +356,12 @@ void CreatePropertyGraphFunction::CreatePropertyGraphFunc(
for (idx_t i = 0; i < e_table->sub_labels.size(); i++) {
insert_info += "'" + e_table->sub_labels[i] + (i == e_table->sub_labels.size() - 1 ? "'" : "', ");
}
insert_info += "]";
insert_info += "], ";
} else {
insert_info += "NULL";
insert_info += "NULL, ";
}
insert_info += "'" + e_table->catalog_name + "', ";
insert_info += "'" + e_table->schema_name + "'";
insert_info += "), ";
}

Expand Down
2 changes: 0 additions & 2 deletions src/core/functions/table/local_clustering_coefficient.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
#include "duckdb/parser/expression/function_expression.hpp"
#include "duckpgq/core/utils/compressed_sparse_row.hpp"
#include <duckdb/parser/query_node/select_node.hpp>
#include <duckdb/parser/tableref/basetableref.hpp>
#include <duckdb/parser/tableref/joinref.hpp>
#include <duckdb/parser/tableref/subqueryref.hpp>
#include <duckpgq/core/functions/table.hpp>

Expand Down
49 changes: 21 additions & 28 deletions src/core/functions/table/match.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ unique_ptr<SubqueryRef> PGQMatchFunction::CreateCountCTESubquery() {
auto count_function =
make_uniq<FunctionExpression>("count", std::move(children));

auto zero = make_uniq<ConstantExpression>(Value::INTEGER((int32_t)0));
auto zero = make_uniq<ConstantExpression>(Value::INTEGER(0));

vector<unique_ptr<ParsedExpression>> multiply_children;

Expand Down Expand Up @@ -190,14 +190,13 @@ void PGQMatchFunction::EdgeTypeAny(
// START SELECT src, dst, * from edge_table
auto src_dst_select_node = make_uniq<SelectNode>();

auto edge_left_ref = make_uniq<BaseTableRef>();
edge_left_ref->table_name = edge_table->table_name;
auto edge_left_ref = edge_table->CreateBaseTableRef(edge_binding);
src_dst_select_node->from_table = std::move(edge_left_ref);
auto src_dst_children = vector<unique_ptr<ParsedExpression>>();
src_dst_children.push_back(make_uniq<ColumnRefExpression>(
edge_table->source_fk[0], edge_table->table_name));
edge_table->source_fk[0], edge_binding));
src_dst_children.push_back(make_uniq<ColumnRefExpression>(
edge_table->destination_fk[0], edge_table->table_name));
edge_table->destination_fk[0], edge_binding));
src_dst_children.push_back(make_uniq<StarExpression>());

src_dst_select_node->select_list = std::move(src_dst_children);
Expand All @@ -206,15 +205,14 @@ void PGQMatchFunction::EdgeTypeAny(
// START SELECT dst, src, * from edge_table
auto dst_src_select_node = make_uniq<SelectNode>();

auto edge_right_ref = make_uniq<BaseTableRef>();
edge_right_ref->table_name = edge_table->table_name;
auto edge_right_ref = edge_table->CreateBaseTableRef(edge_binding);
auto dst_src_children = vector<unique_ptr<ParsedExpression>>();
dst_src_select_node->from_table = std::move(edge_right_ref);

dst_src_children.push_back(make_uniq<ColumnRefExpression>(
edge_table->destination_fk[0], edge_table->table_name));
edge_table->destination_fk[0], edge_binding));
dst_src_children.push_back(make_uniq<ColumnRefExpression>(
edge_table->source_fk[0], edge_table->table_name));
edge_table->source_fk[0], edge_binding));
dst_src_children.push_back(make_uniq<StarExpression>());

dst_src_select_node->select_list = std::move(dst_src_children);
Expand Down Expand Up @@ -289,7 +287,7 @@ void PGQMatchFunction::EdgeTypeLeftRight(
const string &edge_binding, const string &prev_binding,
const string &next_binding,
vector<unique_ptr<ParsedExpression>> &conditions,
unordered_map<string, string> &alias_map, int32_t &extra_alias_counter) {
case_insensitive_map_t<shared_ptr<PropertyGraphTable>> &alias_map, int32_t &extra_alias_counter) {
auto src_left_expr = CreateMatchJoinExpression(
edge_table->source_pk, edge_table->source_fk, next_binding, edge_binding);
auto dst_left_expr = CreateMatchJoinExpression(edge_table->destination_pk,
Expand All @@ -304,7 +302,7 @@ void PGQMatchFunction::EdgeTypeLeftRight(
edge_binding + std::to_string(extra_alias_counter);
extra_alias_counter++;

alias_map[additional_edge_alias] = edge_table->table_name;
alias_map[additional_edge_alias] = edge_table;

auto src_right_expr =
CreateMatchJoinExpression(edge_table->source_pk, edge_table->source_fk,
Expand Down Expand Up @@ -345,7 +343,7 @@ PGQMatchFunction::CreateWhereClause(vector<unique_ptr<ParsedExpression>> &condit
}

unique_ptr<CommonTableExpressionInfo> PGQMatchFunction::GenerateShortestPathCTE(CreatePropertyGraphInfo &pg_table, SubPath *edge_subpath,
PathElement * previous_vertex_element, PathElement * next_vertex_element, vector<unique_ptr<ParsedExpression>> &path_finding_conditions) {
PathElement *previous_vertex_element, PathElement * next_vertex_element, vector<unique_ptr<ParsedExpression>> &path_finding_conditions) {
auto cte_info = make_uniq<CommonTableExpressionInfo>();
auto select_statement = make_uniq<SelectStatement>();
auto select_node = make_uniq<SelectNode>();
Expand All @@ -362,7 +360,7 @@ unique_ptr<CommonTableExpressionInfo> PGQMatchFunction::GenerateShortestPathCTE(
vector<unique_ptr<ParsedExpression>> pathfinding_children;
pathfinding_children.push_back(std::move(csr_id));
pathfinding_children.push_back(std::move(GetCountTable(
edge_table->source_reference, previous_vertex_element->variable_binding, edge_table->source_pk[0])));
edge_table->source_pg_table, previous_vertex_element->variable_binding, edge_table->source_pk[0])));
pathfinding_children.push_back(std::move(src_row_id));
pathfinding_children.push_back(std::move(dst_row_id));

Expand All @@ -377,11 +375,9 @@ unique_ptr<CommonTableExpressionInfo> PGQMatchFunction::GenerateShortestPathCTE(
dst_rowid_outer_select->alias = "dst_rowid";
select_node->select_list.push_back(std::move(dst_rowid_outer_select));

auto src_tableref = make_uniq<BaseTableRef>();
src_tableref->table_name = edge_table->source_reference;
auto src_tableref = edge_table->source_pg_table->CreateBaseTableRef();
src_tableref->alias = previous_vertex_element->variable_binding;
auto dst_tableref = make_uniq<BaseTableRef>();
dst_tableref->table_name = edge_table->destination_reference;
auto dst_tableref = edge_table->destination_pg_table->CreateBaseTableRef();
dst_tableref->alias = next_vertex_element->variable_binding;
auto first_cross_join_ref = make_uniq<JoinRef>(JoinRefType::CROSS);
first_cross_join_ref->left = std::move(src_tableref);
Expand Down Expand Up @@ -560,10 +556,10 @@ void PGQMatchFunction::AddEdgeJoins(
PGQMatchType edge_type, const string &edge_binding,
const string &prev_binding, const string &next_binding,
vector<unique_ptr<ParsedExpression>> &conditions,
unordered_map<string, string> &alias_map, int32_t &extra_alias_counter,
case_insensitive_map_t<shared_ptr<PropertyGraphTable>> &alias_map, int32_t &extra_alias_counter,
unique_ptr<TableRef> &from_clause) {
if (edge_type != PGQMatchType::MATCH_EDGE_ANY) {
alias_map[edge_binding] = edge_table->table_name;
alias_map[edge_binding] = edge_table;
}
switch (edge_type) {
case PGQMatchType::MATCH_EDGE_ANY: {
Expand Down Expand Up @@ -602,7 +598,7 @@ unique_ptr<ParsedExpression> PGQMatchFunction::AddPathQuantifierCondition(
vector<unique_ptr<ParsedExpression>> pathfinding_children;
pathfinding_children.push_back(std::move(csr_id));
pathfinding_children.push_back(
std::move(GetCountTable(edge_table->source_reference, prev_binding, edge_table->source_pk[0])));
std::move(GetCountTable(edge_table->source_pg_table, prev_binding, edge_table->source_pk[0])));
pathfinding_children.push_back(std::move(src_row_id));
pathfinding_children.push_back(std::move(dst_row_id));

Expand Down Expand Up @@ -773,7 +769,7 @@ void PGQMatchFunction::ProcessPathList(
vector<unique_ptr<PathReference>> &path_list,
vector<unique_ptr<ParsedExpression>> &conditions,
unique_ptr<SelectNode> &final_select_node,
unordered_map<string, string> &alias_map, CreatePropertyGraphInfo &pg_table,
case_insensitive_map_t<shared_ptr<PropertyGraphTable>> &alias_map, CreatePropertyGraphInfo &pg_table,
int32_t &extra_alias_counter,
MatchExpression &original_ref) {
PathElement *previous_vertex_element = GetPathElement(path_list[0]);
Expand Down Expand Up @@ -801,7 +797,7 @@ void PGQMatchFunction::ProcessPathList(
FindGraphTable(previous_vertex_element->label, pg_table);
CheckInheritance(previous_vertex_table, previous_vertex_element, conditions);
alias_map[previous_vertex_element->variable_binding] =
previous_vertex_table->table_name;
previous_vertex_table;

for (idx_t idx_j = 1; idx_j < path_list.size(); idx_j = idx_j + 2) {
PathElement *next_vertex_element = GetPathElement(path_list[idx_j + 1]);
Expand All @@ -824,8 +820,7 @@ void PGQMatchFunction::ProcessPathList(
auto next_vertex_table =
FindGraphTable(next_vertex_element->label, pg_table);
CheckInheritance(next_vertex_table, next_vertex_element, conditions);
alias_map[next_vertex_element->variable_binding] =
next_vertex_table->table_name;
alias_map[next_vertex_element->variable_binding] = next_vertex_table;

PathElement *edge_element = GetPathElement(path_list[idx_j]);
if (!edge_element) {
Expand Down Expand Up @@ -892,7 +887,7 @@ PGQMatchFunction::MatchBindReplace(ClientContext &context,
vector<unique_ptr<ParsedExpression>> conditions;

auto final_select_node = make_uniq<SelectNode>();
unordered_map<string, string> alias_map;
case_insensitive_map_t<shared_ptr<PropertyGraphTable>> alias_map;

int32_t extra_alias_counter = 0;
for (idx_t idx_i = 0; idx_i < ref->path_patterns.size(); idx_i++) {
Expand All @@ -906,10 +901,8 @@ PGQMatchFunction::MatchBindReplace(ClientContext &context,

// Go through all aliases encountered
for (auto &table_alias_entry : alias_map) {
auto table_ref = make_uniq<BaseTableRef>();
table_ref->table_name = table_alias_entry.second;
auto table_ref = table_alias_entry.second->CreateBaseTableRef();
table_ref->alias = table_alias_entry.first;

if (final_select_node->from_table) {
auto new_root = make_uniq<JoinRef>(JoinRefType::CROSS);
new_root->left = std::move(final_select_node->from_table);
Expand Down
Loading
Loading