diff --git a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb index 8ef3439407..ca47cef547 100644 --- a/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featuregroup_spec.rb @@ -1676,41 +1676,6 @@ expect(parsed_json.first["features"][0].key?("defaultValue")).to be false expect(parsed_json.first["features"][0].key?("featureGroupId")).to be true end - - describe "with quota enabled" do - before :all do - setVar("quotas_featuregroups_online_disabled", "1") - setVar("quotas_featuregroups_online_enabled", "1") - end - after :all do - setVar("quotas_featuregroups_online_disabled", "-1") - setVar("quotas_featuregroups_online_enabled", "-1") - end - it "should fail to create cached feature groups if quota has been reached" do - ## Create new project - project = create_project - featurestore_id = get_featurestore_id(project.id) - ## First attempt should succeed - create_cached_featuregroup(project.id, featurestore_id, online: true) - expect_status_details(201) - - ## This time is should fail because it has reached the online enabled limit - result, _ =create_cached_featuregroup(project.id, featurestore_id, online: true) - expect_status_details(500) - parsed = JSON.parse(result) - expect(parsed['devMsg']).to include("quota") - - ## Online disabled should go through - create_cached_featuregroup(project.id, featurestore_id, online: false) - expect_status_details(201) - - ## Now reached limit for online disabled too - result, _ = create_cached_featuregroup(project.id, featurestore_id, online: false) - expect_status_details(500) - parsed = JSON.parse(result) - expect(parsed['devMsg']).to include("quota") - end - end end end @@ -3583,418 +3548,4 @@ end end end - - describe "shared permissions" do - before :all do - # Create users - @user1_params = {email: "user1_#{random_id}@email.com", first_name: "User", last_name: "1", password: "Pass123"} - @user1 = create_user(@user1_params) - pp "user email: #{@user1[:email]}" if defined?(@debugOpt) && @debugOpt - @user2_params = {email: "user2_#{random_id}@email.com", first_name: "User", last_name: "2", password: "Pass123"} - @user2 = create_user(@user2_params) - pp "user email: #{@user2[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_scientist_params = {email: "data_scientist_#{random_id}@email.com", first_name: "User", last_name: "data_scientist", password: "Pass123"} - @user_data_scientist = create_user(@user_data_scientist_params) - pp "user email: #{@user_data_scientist[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_owner_params = {email: "data_owner_#{random_id}@email.com", first_name: "User", last_name: "data_owner", password: "Pass123"} - @user_data_owner = create_user(@user_data_owner_params) - pp "user email: #{@user_data_owner[:email]}" if defined?(@debugOpt) && @debugOpt - - # Create base project - create_session(@user1[:email], @user1_params[:password]) - @project1 = create_project - pp @project1[:projectname] if defined?(@debugOpt) && @debugOpt - with_jdbc_connector(@project1[:id]) - - # Create shared with projects - create_session(@user2[:email], @user2_params[:password]) - @project_read_only = create_project - pp @project_read_only[:projectname] if defined?(@debugOpt) && @debugOpt - - # Add members to projects - add_member_to_project(@project_read_only, @user_data_owner_params[:email], "Data owner") - add_member_to_project(@project_read_only, @user_data_scientist_params[:email], "Data scientist") - - # Share projects - create_session(@user1[:email], @user1_params[:password]) - share_dataset_checked(@project1, "#{@project1[:projectname].downcase}_featurestore.db", @project_read_only[:projectname], datasetType: "FEATURESTORE") - - # Accept shared projects - create_session(@user2[:email], @user2_params[:password]) - accept_dataset_checked(@project_read_only, "#{@project1[:projectname]}::#{@project1[:projectname].downcase}_featurestore.db", datasetType: "FEATURESTORE") - end - - context "shared cached feature group permissions" do - - # create - - it 'data owner should not be able to create fg with read only permission' do - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - json_result, _ = create_cached_featuregroup(@project_read_only[:id], fs["featurestoreId"], online:true) - expect_status_details(403) - end - - it 'data scientist should not be able to create fg with read only permission' do - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - json_result, _ = create_cached_featuregroup(@project_read_only[:id], fs["featurestoreId"], online:true) - expect_status_details(403) - end - - # get - - it 'data owner should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - _, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - it 'data scientist should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - _, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - # update - - it 'data owner should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - new_description = "changed description" - update_cached_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - it 'data scientist should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - new_description = "changed description" - update_cached_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - # delete - - it 'data owner should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - - it 'data scientist should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - end - - context "shared stream feature group permission" do - - # create - - it 'data owner should not be able to create fg with read only permission' do - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) - expect_status_details(403) - end - - it 'data scientist should not be able to create fg with read only permission' do - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) - expect_status_details(403) - end - - # get - - it 'data owner should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - _, fg_name = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - it 'data scientist should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - _, fg_name = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - # update - - it 'data owner should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - new_description = "changed description" - update_stream_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - it 'data scientist should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - new_description = "changed description" - update_stream_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - # delete - - it 'data owner should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - - it 'data scientist should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - - # get subject - - it 'data owner should be able to get fg schema' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) - expect_status_details(200) - end - - it 'data scientist should be able to get fg schema' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) - expect_status_details(200) - end - - it 'user should not be able to get fg schema from project that is not shared' do - create_session(@user2[:email], @user2_params[:password]) - fs = get_featurestore(@project_read_only[:id]) - json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user1[:email], @user1_params[:password]) - get_featurestore_subject_details(@project1, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) - expect_status_details(404) - end - end - - context "shared on demand feature group permission" do - - # create - - it 'data owner should not be able to create fg with read only permission' do - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - connector_id = get_jdbc_connector_id - create_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id) - expect_status_details(403) - end - - it 'data scientist should not be able to create fg with read only permission' do - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) - connector_id = get_jdbc_connector_id - create_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id) - expect_status_details(403) - end - - # get - - it 'data owner should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - _, fg_name = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - it 'data scientist should be able to get fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - _, fg_name = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) - end - - # update - - it 'data owner should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - new_description = "changed description" - update_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id, fg[:id], fg[:version], featuregroup_desc: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - it 'data scientist should not be able to update fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - new_description = "changed description" - update_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id, fg[:id], fg[:version], featuregroup_desc: new_description) - expect_status_details(403) - - new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - expect(new_fg.length).to be 1 - new_fg = new_fg[0] - expect(new_fg["description"]).not_to eql(new_description) - end - - # delete - - it 'data owner should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - - it 'data scientist should not be able to delete fg with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - connector_id = get_jdbc_connector_id - json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) - expect_status_details(201) - fg = JSON.parse(json_result, :symbolize_names => true) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) - expect_status_details(403) - - get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) - end - end - end end diff --git a/hopsworks-IT/src/test/ruby/spec/featurestore_share_spec.rb b/hopsworks-IT/src/test/ruby/spec/featurestore_share_spec.rb new file mode 100644 index 0000000000..d746c41190 --- /dev/null +++ b/hopsworks-IT/src/test/ruby/spec/featurestore_share_spec.rb @@ -0,0 +1,830 @@ +=begin + This file is part of Hopsworks + Copyright (C) 2024, Hopsworks AB. All rights reserved + + Hopsworks is free software: you can redistribute it and/or modify it under the terms of + the GNU Affero General Public License as published by the Free Software Foundation, + either version 3 of the License, or (at your option) any later version. + + Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. + If not, see . +=end + +describe "On #{ENV['OS']}" do + after(:all) {clean_all_test_projects(spec: "featurestore_share")} + describe 'featurestore' do + describe "shared permissions" do + before :all do + # Create users + @user1_params = {email: "user1_#{random_id}@email.com", first_name: "User", last_name: "1", password: "Pass123"} + @user1 = create_user(@user1_params) + pp "user email: #{@user1[:email]}" if defined?(@debugOpt) && @debugOpt + @user2_params = {email: "user2_#{random_id}@email.com", first_name: "User", last_name: "2", password: "Pass123"} + @user2 = create_user(@user2_params) + pp "user email: #{@user2[:email]}" if defined?(@debugOpt) && @debugOpt + @user_data_scientist_params = {email: "data_scientist_#{random_id}@email.com", first_name: "User", last_name: "data_scientist", password: "Pass123"} + @user_data_scientist = create_user(@user_data_scientist_params) + pp "user email: #{@user_data_scientist[:email]}" if defined?(@debugOpt) && @debugOpt + @user_data_owner_params = {email: "data_owner_#{random_id}@email.com", first_name: "User", last_name: "data_owner", password: "Pass123"} + @user_data_owner = create_user(@user_data_owner_params) + pp "user email: #{@user_data_owner[:email]}" if defined?(@debugOpt) && @debugOpt + + # Create base project + create_session(@user1[:email], @user1_params[:password]) + @project1 = create_project + pp @project1[:projectname] if defined?(@debugOpt) && @debugOpt + with_jdbc_connector(@project1[:id]) + + # Create shared with projects + create_session(@user2[:email], @user2_params[:password]) + @project_read_only = create_project + pp @project_read_only[:projectname] if defined?(@debugOpt) && @debugOpt + + @featurestore_id = get_featurestore_id(@project_read_only[:id]) + json_result, _ = create_cached_featuregroup(@project_read_only[:id], @featurestore_id) + expect_status_details(201) + @cached_feature_group = JSON.parse(json_result) + create_statistics_commit_fg(@project_read_only[:id], @featurestore_id, @cached_feature_group["id"]) + + expectation_suite = generate_template_expectation_suite() + @expectation_dto = create_expectation_suite(@project_read_only[:id], @featurestore_id, + @cached_feature_group["id"], expectation_suite) + + # Add members to projects + add_member_to_project(@project_read_only, @user_data_owner_params[:email], "Data owner") + add_member_to_project(@project_read_only, @user_data_scientist_params[:email], "Data scientist") + + # Share projects + create_session(@user1[:email], @user1_params[:password]) + share_dataset_checked(@project1, "#{@project1[:projectname].downcase}_featurestore.db", @project_read_only[:projectname], datasetType: "FEATURESTORE") + + # Accept shared projects + create_session(@user2[:email], @user2_params[:password]) + accept_dataset_checked(@project_read_only, "#{@project1[:projectname]}::#{@project1[:projectname].downcase}_featurestore.db", datasetType: "FEATURESTORE") + end + + context "shared cached feature group permissions" do + + # create + + it 'data owner should not be able to create fg with read only permission' do + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + json_result, _ = create_cached_featuregroup(@project_read_only[:id], fs["featurestoreId"], online:true) + expect_status_details(403) + end + + it 'data scientist should not be able to create fg with read only permission' do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + json_result, _ = create_cached_featuregroup(@project_read_only[:id], fs["featurestoreId"], online:true) + expect_status_details(403) + end + + # get + + it 'data owner should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + _, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + it 'data scientist should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + _, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + # update + + it 'data owner should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + new_description = "changed description" + update_cached_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + it 'data scientist should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + new_description = "changed description" + update_cached_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + # delete + + it 'data owner should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + + it 'data scientist should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + end + + context "shared stream feature group permission" do + + # create + + it 'data owner should not be able to create fg with read only permission' do + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) + expect_status_details(403) + end + + it 'data scientist should not be able to create fg with read only permission' do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) + expect_status_details(403) + end + + # get + + it 'data owner should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + _, fg_name = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + it 'data scientist should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + _, fg_name = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + # update + + it 'data owner should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + new_description = "changed description" + update_stream_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + it 'data scientist should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + new_description = "changed description" + update_stream_featuregroup_metadata(@project_read_only[:id], fs["featurestoreId"], fg[:id], fg[:version], description: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + # delete + + it 'data owner should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + + it 'data scientist should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + + # get subject + + it 'data owner should be able to get fg schema' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) + expect_status_details(200) + end + + it 'data scientist should be able to get fg schema' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, _ = create_stream_featuregroup(@project1[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_featurestore_subject_details(@project_read_only, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) + expect_status_details(200) + end + + it 'user should not be able to get fg schema from project that is not shared' do + create_session(@user2[:email], @user2_params[:password]) + fs = get_featurestore(@project_read_only[:id]) + json_result, _ = create_stream_featuregroup(@project_read_only[:id], fs["featurestoreId"]) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user1[:email], @user1_params[:password]) + get_featurestore_subject_details(@project1, fs["featurestoreId"], "#{fg[:name]}_#{fg[:version]}", 1) + expect_status_details(404) + end + end + + context "shared on demand feature group permission" do + + # create + + it 'data owner should not be able to create fg with read only permission' do + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + connector_id = get_jdbc_connector_id + create_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id) + expect_status_details(403) + end + + it 'data scientist should not be able to create fg with read only permission' do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + fs = get_featurestore(@project_read_only[:id], fs_project_id: @project1[:id]) + connector_id = get_jdbc_connector_id + create_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id) + expect_status_details(403) + end + + # get + + it 'data owner should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + _, fg_name = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + it 'data scientist should be able to get fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + _, fg_name = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_featuregroup_checked(@project_read_only[:id], fg_name, fs_id: fs["featurestoreId"]) + end + + # update + + it 'data owner should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + new_description = "changed description" + update_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id, fg[:id], fg[:version], featuregroup_desc: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + it 'data scientist should not be able to update fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + new_description = "changed description" + update_on_demand_featuregroup(@project_read_only[:id], fs["featurestoreId"], connector_id, fg[:id], fg[:version], featuregroup_desc: new_description) + expect_status_details(403) + + new_fg = get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + expect(new_fg.length).to be 1 + new_fg = new_fg[0] + expect(new_fg["description"]).not_to eql(new_description) + end + + # delete + + it 'data owner should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + + it 'data scientist should not be able to delete fg with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + connector_id = get_jdbc_connector_id + json_result, _ = create_on_demand_featuregroup(@project1[:id], fs["featurestoreId"], connector_id) + expect_status_details(201) + fg = JSON.parse(json_result, :symbolize_names => true) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + delete_featuregroup(@project_read_only[:id], fs["featurestoreId"], fg[:id]) + expect_status_details(403) + + get_featuregroup_checked(@project_read_only[:id], fg[:name], fs_id: fs["featurestoreId"]) + end + end + + context "shared training dataset permissions" do + + # create + + it 'data owner should not be able to create td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + json_result, training_dataset_name = create_hopsfs_training_dataset(@project_read_only[:id], fs["featurestoreId"], connector) + expect_status_details(403) + end + + it 'data scientist should not be able to create td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + json_result, training_dataset_name = create_hopsfs_training_dataset(@project_read_only[:id], fs["featurestoreId"], connector) + expect_status_details(403) + end + + # get + + it 'data owner should be able to get td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + get_training_datasets_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{@project_read_only[:id]}/featurestores/#{fs["featurestoreId"]}/trainingdatasets/#{training_dataset_name}" + get get_training_datasets_endpoint + parsed_json = JSON.parse(response.body) + expect_status_details(200) + expect(parsed_json.size).to eq 1 + end + + it 'data scientist should be able to get td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_training_datasets_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{@project_read_only[:id]}/featurestores/#{fs["featurestoreId"]}/trainingdatasets/#{training_dataset_name}" + get get_training_datasets_endpoint + parsed_json = JSON.parse(response.body) + expect_status_details(200) + expect(parsed_json.size).to eq 1 + end + + # update + + it 'data owner should not be able to update td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + json_result2 = update_hopsfs_training_dataset_metadata(@project_read_only[:id], fs["featurestoreId"], parsed_json["id"], "tfrecords", connector) + expect_status_details(403) + end + + it 'data scientist should not be able to update td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + json_result2 = update_hopsfs_training_dataset_metadata(@project_read_only[:id], fs["featurestoreId"], parsed_json["id"], "tfrecords", connector) + expect_status_details(403) + end + + # delete + + it 'data owner should not be able to delete td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + delete_training_dataset_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + @project_read_only["id"].to_s + + "/featurestores/" + fs["featurestoreId"].to_s + "/trainingdatasets/" + parsed_json["id"].to_s + delete delete_training_dataset_endpoint + expect_status_details(403) + + # make sure inode is deleted + path = "/Projects/#{@project1['projectname']}/#{@project1['projectname']}_Training_Datasets/#{training_dataset_name}_1" + expect(test_dir(path)).to be true + end + + it 'data scientist should not be able to delete td with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) + + json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + delete_training_dataset_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + @project_read_only["id"].to_s + + "/featurestores/" + fs["featurestoreId"].to_s + "/trainingdatasets/" + parsed_json["id"].to_s + delete delete_training_dataset_endpoint + expect_status_details(403) + + # make sure inode is deleted + path = "/Projects/#{@project1['projectname']}/#{@project1['projectname']}_Training_Datasets/#{training_dataset_name}_1" + expect(test_dir(path)).to be true + end + end + + context "shared feature view permissions" do + + # create + + it 'data owner should not be able to create fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + create_feature_view_from_feature_group(@project_read_only[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(403) + end + + it 'data scientist should not be able to create fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + create_feature_view_from_feature_group(@project_read_only[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(403) + end + + # get + + it 'data owner should be able to get fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + end + + it 'data scientist should be able to get fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + end + + # update + + it 'data owner should not be able to update fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + description = parsed_json["description"] + + json_data = { + name: "new_testfeatureviewname", + version: parsed_json["version"], + type: "featureViewDTO", + description: "temp desc" + } + json_result = update_feature_view(@project_read_only[:id], fs["featurestoreId"], json_data, parsed_json["name"], parsed_json["version"]) + expect_status_details(403) + + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + expect(parsed_json["description"]).to eql(description) + end + + it 'data scientist should not be able to update fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + description = parsed_json["description"] + + json_data = { + name: "new_testfeatureviewname", + version: parsed_json["version"], + type: "featureViewDTO", + description: "temp desc" + } + json_result = update_feature_view(@project_read_only[:id], fs["featurestoreId"], json_data, parsed_json["name"], parsed_json["version"]) + expect_status_details(403) + + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + expect(parsed_json["description"]).to eql(description) + end + + # delete + + it 'data owner should not be able to delete fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + delete_feature_view(@project_read_only["id"], parsed_json["name"], feature_store_id: fs["featurestoreId"], expected_status: 403) + + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + end + + it 'data scientist should not be able to delete fv with read only permission' do + create_session(@user1[:email], @user1_params[:password]) + fs = get_featurestore(@project1[:id]) + json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) + parsed_json = JSON.parse(json_result) + expect_status_details(201) + + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + delete_feature_view(@project_read_only["id"], parsed_json["name"], feature_store_id: fs["featurestoreId"], expected_status: 403) + + json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) + parsed_json = JSON.parse(json_result) + expect_status_details(200) + end + end + + context "shared statistics dataset permissions" do + # get + it 'data owner should be able to get statistics with read only permission' do + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + json_result = get_statistics_commit_fg(@project_read_only[:id], @featurestore_id, @cached_feature_group["id"]) + expect_status_details(200) + end + + it 'data scientist should be able to get statistics with read only permission' do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + json_result = get_statistics_commit_fg(@project_read_only[:id], @featurestore_id, @cached_feature_group["id"]) + expect_status_details(200) + end + + it "should get stat of a shared statistics dataset" do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_dataset_stat(@project_read_only, "/Projects/#{@project1[:projectname]}/Statistics") + expect_status_details(200) + end + + it "should fail to create dir in a shared statistics dataset" do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + create_dir(@project_read_only, "/Projects/#{@project1[:projectname]}/Statistics/test") + expect_status_details(403, error_code: 200002) + end + + end + + context "shared validation dataset permissions" do + # get + it 'data owner should be able to get validation with read only permission' do + create_session(@user_data_owner[:email], @user_data_owner_params[:password]) + json_result = get_expectation_suite(@project_read_only[:id], @featurestore_id, @cached_feature_group["id"]) + expect_status_details(200) + end + + it 'data scientist should be able to get validation with read only permission' do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + json_result = get_expectation_suite(@project_read_only[:id], @featurestore_id, @cached_feature_group["id"]) + expect_status_details(200) + end + + it "should get stat of a shared DataValidation dataset" do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + get_dataset_stat(@project_read_only, "/Projects/#{@project1[:projectname]}/DataValidation") + expect_status_details(200) + end + + it "should fail to create dir in a shared DataValidation dataset" do + create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) + create_dir(@project_read_only, "/Projects/#{@project1[:projectname]}/DataValidation/test") + expect_status_details(403, error_code: 200002) + end + + end + end + end +end \ No newline at end of file diff --git a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb index 8d956ea02f..6eb6d535b6 100644 --- a/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/featureview_spec.rb @@ -846,212 +846,5 @@ end end end - - describe "shared permissions" do - before :all do - # Create users - @user1_params = {email: "user1_#{random_id}@email.com", first_name: "User", last_name: "1", password: "Pass123"} - @user1 = create_user(@user1_params) - pp "user email: #{@user1[:email]}" if defined?(@debugOpt) && @debugOpt - @user2_params = {email: "user2_#{random_id}@email.com", first_name: "User", last_name: "2", password: "Pass123"} - @user2 = create_user(@user2_params) - pp "user email: #{@user2[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_scientist_params = {email: "data_scientist_#{random_id}@email.com", first_name: "User", last_name: "data_scientist", password: "Pass123"} - @user_data_scientist = create_user(@user_data_scientist_params) - pp "user email: #{@user_data_scientist[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_owner_params = {email: "data_owner_#{random_id}@email.com", first_name: "User", last_name: "data_owner", password: "Pass123"} - @user_data_owner = create_user(@user_data_owner_params) - pp "user email: #{@user_data_owner[:email]}" if defined?(@debugOpt) && @debugOpt - - # Create base project - create_session(@user1[:email], @user1_params[:password]) - @project1 = create_project - pp @project1[:projectname] if defined?(@debugOpt) && @debugOpt - - # Create shared with projects - create_session(@user2[:email], @user2_params[:password]) - @project_read_only = create_project - pp @project_read_only[:projectname] if defined?(@debugOpt) && @debugOpt - - # Add members to projects - add_member_to_project(@project_read_only, @user_data_owner_params[:email], "Data owner") - add_member_to_project(@project_read_only, @user_data_scientist_params[:email], "Data scientist") - - # Share projects - create_session(@user1[:email], @user1_params[:password]) - share_dataset_checked(@project1, "#{@project1[:projectname].downcase}_featurestore.db", @project_read_only[:projectname], datasetType: "FEATURESTORE") - - # Accept shared projects - create_session(@user2[:email], @user2_params[:password]) - accept_dataset_checked(@project_read_only, "#{@project1[:projectname]}::#{@project1[:projectname].downcase}_featurestore.db", datasetType: "FEATURESTORE") - end - - context "shared feature view permissions" do - - # create - - it 'data owner should not be able to create fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - create_feature_view_from_feature_group(@project_read_only[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(403) - end - - it 'data scientist should not be able to create fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - create_feature_view_from_feature_group(@project_read_only[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(403) - end - - # get - - it 'data owner should be able to get fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - end - - it 'data scientist should be able to get fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - end - - # update - - it 'data owner should not be able to update fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - description = parsed_json["description"] - - json_data = { - name: "new_testfeatureviewname", - version: parsed_json["version"], - type: "featureViewDTO", - description: "temp desc" - } - json_result = update_feature_view(@project_read_only[:id], fs["featurestoreId"], json_data, parsed_json["name"], parsed_json["version"]) - expect_status_details(403) - - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - expect(parsed_json["description"]).to eql(description) - end - - it 'data scientist should not be able to update fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - description = parsed_json["description"] - - json_data = { - name: "new_testfeatureviewname", - version: parsed_json["version"], - type: "featureViewDTO", - description: "temp desc" - } - json_result = update_feature_view(@project_read_only[:id], fs["featurestoreId"], json_data, parsed_json["name"], parsed_json["version"]) - expect_status_details(403) - - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - expect(parsed_json["description"]).to eql(description) - end - - # delete - - it 'data owner should not be able to delete fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - delete_feature_view(@project_read_only["id"], parsed_json["name"], feature_store_id: fs["featurestoreId"], expected_status: 403) - - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - end - - it 'data scientist should not be able to delete fv with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - json_result = create_feature_view_from_feature_group(@project1[:id], fs["featurestoreId"], parsed_json) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - delete_feature_view(@project_read_only["id"], parsed_json["name"], feature_store_id: fs["featurestoreId"], expected_status: 403) - - json_result = get_feature_view_by_name_and_version(@project_read_only[:id], fs["featurestoreId"], parsed_json["name"], parsed_json["version"]) - parsed_json = JSON.parse(json_result) - expect_status_details(200) - end - end - end end end diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/dataset_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/dataset_helper.rb index 7a8dbda073..5d0a3475cc 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/dataset_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/dataset_helper.rb @@ -218,7 +218,7 @@ def delete_dir(project, path, dataset_type: "DATASET") end def delete_dir_checked(project, path, dataset_type: "DATASET") - delete_dir(project, path, dataset_type) + delete_dir(project, path, dataset_type: dataset_type) expect_status_details(204) end @@ -847,4 +847,27 @@ def query_dataset(project, path, dataset_type: "DATASET", action: "stat", expand pp "get #{path}" if defined?(@debugOpt) && @debugOpt get path end + + def test_feature_store_shared(project1, project2) + featurestore = "#{project1[:projectname]}::#{project1[:projectname].downcase}_featurestore.db" + trainingDataset = "#{project1[:projectname]}::#{project1[:projectname]}_Training_Datasets" + statisticsDataset = "#{project1[:projectname]}::Statistics" + dataValidationDataset = "#{project1[:projectname]}::DataValidation" + + get_dataset_stat(project2, featurestore, datasetType: "&type=FEATURESTORE") + shared_ds = json_body + expect("#{shared_ds[:name]}").to eq(featurestore) + + get_dataset_stat(project2, trainingDataset, datasetType: "&type=DATASET") + shared_ds = json_body + expect("#{shared_ds[:name]}").to eq(trainingDataset) + + get_dataset_stat(project2, statisticsDataset, datasetType: "&type=DATASET") + shared_ds = json_body + expect("#{shared_ds[:name]}").to eq(statisticsDataset) + + get_dataset_stat(project2, dataValidationDataset, datasetType: "&type=DATASET") + shared_ds = json_body + expect("#{shared_ds[:name]}").to eq(dataValidationDataset) + end end diff --git a/hopsworks-IT/src/test/ruby/spec/helpers/variables_helper.rb b/hopsworks-IT/src/test/ruby/spec/helpers/variables_helper.rb index 01a4a4c15f..f63e014a17 100644 --- a/hopsworks-IT/src/test/ruby/spec/helpers/variables_helper.rb +++ b/hopsworks-IT/src/test/ruby/spec/helpers/variables_helper.rb @@ -84,6 +84,19 @@ def setVar (id, value) variables end + def create_var(id, value) + variables = Variables.find_by(id: id) + if variables.nil? + Variables.create(id: id, value: value) + variables = Variables.find_by(id: id) + else + variables.value = value + variables.save + end + refresh_variables + variables + end + def kubernetes_installed getVar("kubernetes_installed").value.eql? "true" end diff --git a/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb b/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb index 7de7926c9c..9db0aaa7ba 100644 --- a/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/trainingdataset_spec.rb @@ -2745,208 +2745,5 @@ end end end - - describe "shared permissions" do - before :all do - # Create users - @user1_params = {email: "user1_#{random_id}@email.com", first_name: "User", last_name: "1", password: "Pass123"} - @user1 = create_user(@user1_params) - pp "user email: #{@user1[:email]}" if defined?(@debugOpt) && @debugOpt - @user2_params = {email: "user2_#{random_id}@email.com", first_name: "User", last_name: "2", password: "Pass123"} - @user2 = create_user(@user2_params) - pp "user email: #{@user2[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_scientist_params = {email: "data_scientist_#{random_id}@email.com", first_name: "User", last_name: "data_scientist", password: "Pass123"} - @user_data_scientist = create_user(@user_data_scientist_params) - pp "user email: #{@user_data_scientist[:email]}" if defined?(@debugOpt) && @debugOpt - @user_data_owner_params = {email: "data_owner_#{random_id}@email.com", first_name: "User", last_name: "data_owner", password: "Pass123"} - @user_data_owner = create_user(@user_data_owner_params) - pp "user email: #{@user_data_owner[:email]}" if defined?(@debugOpt) && @debugOpt - - # Create base project - create_session(@user1[:email], @user1_params[:password]) - @project1 = create_project - pp @project1[:projectname] if defined?(@debugOpt) && @debugOpt - - # Create shared with projects - create_session(@user2[:email], @user2_params[:password]) - @project_read_only = create_project - pp @project_read_only[:projectname] if defined?(@debugOpt) && @debugOpt - - # Add members to projects - add_member_to_project(@project_read_only, @user_data_owner_params[:email], "Data owner") - add_member_to_project(@project_read_only, @user_data_scientist_params[:email], "Data scientist") - - # Share projects - create_session(@user1[:email], @user1_params[:password]) - share_dataset_checked(@project1, "#{@project1[:projectname].downcase}_featurestore.db", @project_read_only[:projectname], datasetType: "FEATURESTORE") - - # Accept shared projects - create_session(@user2[:email], @user2_params[:password]) - accept_dataset_checked(@project_read_only, "#{@project1[:projectname]}::#{@project1[:projectname].downcase}_featurestore.db", datasetType: "FEATURESTORE") - end - - context "shared training dataset permissions" do - - # create - - it 'data owner should not be able to create td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - json_result, training_dataset_name = create_hopsfs_training_dataset(@project_read_only[:id], fs["featurestoreId"], connector) - expect_status_details(403) - end - - it 'data scientist should not be able to create td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - json_result, training_dataset_name = create_hopsfs_training_dataset(@project_read_only[:id], fs["featurestoreId"], connector) - expect_status_details(403) - end - - # get - - it 'data owner should be able to get td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - get_training_datasets_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{@project_read_only[:id]}/featurestores/#{fs["featurestoreId"]}/trainingdatasets/#{training_dataset_name}" - get get_training_datasets_endpoint - parsed_json = JSON.parse(response.body) - expect_status_details(200) - expect(parsed_json.size).to eq 1 - end - - it 'data scientist should be able to get td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - get_training_datasets_endpoint = "#{ENV['HOPSWORKS_API']}/project/#{@project_read_only[:id]}/featurestores/#{fs["featurestoreId"]}/trainingdatasets/#{training_dataset_name}" - get get_training_datasets_endpoint - parsed_json = JSON.parse(response.body) - expect_status_details(200) - expect(parsed_json.size).to eq 1 - end - - # update - - it 'data owner should not be able to update td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - json_result2 = update_hopsfs_training_dataset_metadata(@project_read_only[:id], fs["featurestoreId"], parsed_json["id"], "tfrecords", connector) - expect_status_details(403) - end - - it 'data scientist should not be able to update td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - json_result2 = update_hopsfs_training_dataset_metadata(@project_read_only[:id], fs["featurestoreId"], parsed_json["id"], "tfrecords", connector) - expect_status_details(403) - end - - # delete - - it 'data owner should not be able to delete td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_owner[:email], @user_data_owner_params[:password]) - delete_training_dataset_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + @project_read_only["id"].to_s + - "/featurestores/" + fs["featurestoreId"].to_s + "/trainingdatasets/" + parsed_json["id"].to_s - delete delete_training_dataset_endpoint - expect_status_details(403) - - # make sure inode is deleted - path = "/Projects/#{@project1['projectname']}/#{@project1['projectname']}_Training_Datasets/#{training_dataset_name}_1" - expect(test_dir(path)).to be true - end - - it 'data scientist should not be able to delete td with read only permission' do - create_session(@user1[:email], @user1_params[:password]) - fs = get_featurestore(@project1[:id]) - json_result, fg_name = create_cached_featuregroup(@project1[:id], fs["featurestoreId"], online:true) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - connector = get_hopsfs_training_datasets_connector(@project1[:projectname]) - - json_result, training_dataset_name = create_hopsfs_training_dataset(@project1[:id], fs["featurestoreId"], connector) - parsed_json = JSON.parse(json_result) - expect_status_details(201) - - create_session(@user_data_scientist[:email], @user_data_scientist_params[:password]) - delete_training_dataset_endpoint = "#{ENV['HOPSWORKS_API']}/project/" + @project_read_only["id"].to_s + - "/featurestores/" + fs["featurestoreId"].to_s + "/trainingdatasets/" + parsed_json["id"].to_s - delete delete_training_dataset_endpoint - expect_status_details(403) - - # make sure inode is deleted - path = "/Projects/#{@project1['projectname']}/#{@project1['projectname']}_Training_Datasets/#{training_dataset_name}_1" - expect(test_dir(path)).to be true - end - end - end end end diff --git a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb index 713dc9813e..4c5b1914b0 100644 --- a/hopsworks-IT/src/test/ruby/spec/variables_spec.rb +++ b/hopsworks-IT/src/test/ruby/spec/variables_spec.rb @@ -219,5 +219,56 @@ expect(parsed['devMsg']).to include("quota") end end + + describe 'default feature store share' do + before(:all) do + with_valid_project + create_var('default_feature_store_project_id', @project[:id]) + reset_session + end + after(:all) do + setVar('default_feature_store_project_id', '') + reset_session + end + + it "should share feature store of default fs project" do + project = create_project + test_feature_store_shared(@project, project) + end + end + describe "with feature groups quota enabled" do + before :all do + setVar("quotas_featuregroups_online_disabled", "1") + setVar("quotas_featuregroups_online_enabled", "1") + end + after :all do + setVar("quotas_featuregroups_online_disabled", "-1") + setVar("quotas_featuregroups_online_enabled", "-1") + end + it "should fail to create cached feature groups if quota has been reached" do + ## Create new project + project = create_project + featurestore_id = get_featurestore_id(project.id) + ## First attempt should succeed + create_cached_featuregroup(project.id, featurestore_id, online: true) + expect_status_details(201) + + ## This time is should fail because it has reached the online enabled limit + result, _ =create_cached_featuregroup(project.id, featurestore_id, online: true) + expect_status_details(500) + parsed = JSON.parse(result) + expect(parsed['devMsg']).to include("quota") + + ## Online disabled should go through + create_cached_featuregroup(project.id, featurestore_id, online: false) + expect_status_details(201) + + ## Now reached limit for online disabled too + result, _ = create_cached_featuregroup(project.id, featurestore_id, online: false) + expect_status_details(500) + parsed = JSON.parse(result) + expect(parsed['devMsg']).to include("quota") + end + end end end diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportBuilder.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportBuilder.java index 8402682162..eec8693b6c 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportBuilder.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/featurestore/datavalidationv2/reports/ValidationReportBuilder.java @@ -64,7 +64,9 @@ public ValidationReportDTO build(UriInfo uriInfo, Project project, Featuregroup featuregroup, ValidationReport validationReport) { ValidationReportDTO dto = new ValidationReportDTO(); uri(dto, uriInfo, project, featuregroup); - Optional validationDatasetOptional = validationReportController.getValidationDataset(project); + //if featuregroup is shared validation dataset is in the featuregroup's project + Optional validationDatasetOptional = + validationReportController.getValidationDataset(featuregroup.getFeaturestore().getProject()); String path = ""; // the validation dataset will probably always be there if we have the validation reports in db if (validationDatasetOptional.isPresent()) { diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java index 56d2da340a..bbf4ea35aa 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/project/ProjectService.java @@ -552,7 +552,9 @@ public Response createProject(ProjectDTO projectDTO, @Context HttpServletRequest HopsSecurityException, FeaturestoreException, OpenSearchException, SchemaException, IOException { Users user = jWTHelper.getUserPrincipal(sc); - projectController.createProject(projectDTO, user); + Project targetProject = projectController.createProject(projectDTO, user); + + datasetController.shareDefaultFeatureStoreIfItExists(targetProject); RESTApiJsonResponse json = new RESTApiJsonResponse(); StringBuilder message = new StringBuilder(); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java index 0428b52507..bcae70b383 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dataset/DatasetController.java @@ -716,28 +716,59 @@ public void zip(Project project, Users user, Path path, Path destPath) throws Da } } - public void share(String targetProjectName, String fullPath, - DatasetAccessPermission permission, Project project, Users user) - throws DatasetException, ProjectException { - + public void share(String targetProjectName, String fullPath, DatasetAccessPermission permission, Project project, + Users user) throws DatasetException, ProjectException { Project targetProject = projectFacade.findByName(targetProjectName); Dataset ds = getByProjectAndFullPath(project, fullPath); + share(targetProject, project, ds, user, permission); + } + + private DatasetSharedWith share(Project targetProject, Project project, Dataset ds, Users user, + DatasetAccessPermission permission) throws ProjectException, DatasetException { if (targetProject == null) { throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "Target project not found."); } DatasetSharedWith datasetSharedWith = shareInternal(targetProject, ds, user, permission); if (DatasetType.FEATURESTORE.equals(ds.getDsType()) && datasetSharedWith.getAccepted()) { Dataset trainingDataset = getTrainingDataset(project); - if (trainingDataset != null) { - try { - shareInternal(targetProject, trainingDataset, user, permission); - } catch (DatasetException de) { - //Dataset already shared nothing to do - } - } + shareFeatureStoreServiceDataset(user, targetProject, permission, trainingDataset); // If we migrate Training Datasets to remove the project prefix, these methods can be reused shareFeatureStoreServiceDataset(user, project, targetProject, permission, Settings.ServiceDataset.STATISTICS); + shareFeatureStoreServiceDataset(user, project, targetProject, permission, Settings.ServiceDataset.DATAVALIDATION); } + return datasetSharedWith; + } + + public void shareDefaultFeatureStoreIfItExists(Project targetProject) { + Integer srcProjectId = settings.getDefaultFeatureStoreProjectId(); + if (srcProjectId != null) { + Project srcProject = projectFacade.find(srcProjectId); + try { + DatasetSharedWith datasetSharedWith = shareFeatureStore(targetProject, srcProject); + acceptShared(targetProject, targetProject.getOwner(), datasetSharedWith); + } catch (Exception e) { + //We should not fail if default feature store can not be shared. + LOGGER.log(Level.WARNING, "Failed to share default feature store. {0}", e.getMessage()); + } + } + } + + public DatasetSharedWith shareFeatureStore(Project targetProject, Project srcProject) + throws FeaturestoreException, ProjectException, DatasetException { + if (srcProject == null) { + throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_NOT_FOUND, Level.FINE, "Source project " + + "was not provided."); + } + Dataset fsDataset = getProjectFeaturestoreDataset(srcProject); + return share(targetProject, srcProject, fsDataset, srcProject.getOwner(), DatasetAccessPermission.READ_ONLY); + } + + private Dataset getProjectFeaturestoreDataset(Project project) throws FeaturestoreException { + return project.getDatasetCollection().stream() + .filter(ds -> ds.getFeatureStore() != null) + .findFirst() + .orElseThrow(() -> new FeaturestoreException(RESTCodes.FeaturestoreErrorCode.FEATURESTORE_NOT_FOUND, + Level.INFO, "Could not find feature store for project: " + project.getName())); } private Dataset getTrainingDataset(Project project) { @@ -750,6 +781,11 @@ private Dataset getTrainingDataset(Project project) { private void shareFeatureStoreServiceDataset(Users user, Project project, Project targetProject, DatasetAccessPermission permission, Settings.ServiceDataset serviceDataset) { Dataset dataset = getFeatureStoreServiceDataset(project, serviceDataset); + shareFeatureStoreServiceDataset(user, targetProject, permission, dataset); + } + + private void shareFeatureStoreServiceDataset(Users user, Project targetProject, DatasetAccessPermission permission, + Dataset dataset) { if (dataset != null) { try { shareInternal(targetProject, dataset, user, permission); @@ -827,19 +863,18 @@ public void acceptShared(Project project, Users user, DatasetSharedWith datasetS datasetSharedWith.getDataset().getProject(), datasetSharedWith.getPermission(), datasetSharedWith.getSharedBy()); - if (trainingDataset != null && !trainingDataset.getAccepted()) { - try { - acceptSharedDs(user, trainingDataset); - } catch (DatasetException de) { - //Dataset not shared or already accepted nothing to do - } - } + acceptSharedFeatureStoreServiceDataset(user, trainingDataset); // If we migrate Training Datasets to remove the project prefix, these methods can be reused acceptSharedFeatureStoreServiceDataset(project, datasetSharedWith, datasetSharedWith.getPermission(), datasetSharedWith.getSharedBy(), user, Settings.ServiceDataset.STATISTICS); + acceptSharedFeatureStoreServiceDataset(project, datasetSharedWith, + datasetSharedWith.getPermission(), + datasetSharedWith.getSharedBy(), + user, + Settings.ServiceDataset.DATAVALIDATION); // Share the online feature store onlineFeaturestoreController.shareOnlineFeatureStore(project, @@ -870,6 +905,10 @@ private void acceptSharedFeatureStoreServiceDataset(Project project, DatasetShar permission, sharedBy, serviceDataset); + acceptSharedFeatureStoreServiceDataset(acceptedBy, dataset); + } + + private void acceptSharedFeatureStoreServiceDataset(Users acceptedBy, DatasetSharedWith dataset) { if (dataset != null && !dataset.getAccepted()) { try { acceptSharedDs(acceptedBy, dataset); @@ -1203,6 +1242,8 @@ public void unshare(Project project, Users user, Dataset dataset, String targetP } unshareFeatureStoreServiceDataset(user, project, targetProject, datasetSharedWith, Settings.ServiceDataset.STATISTICS, dfso); + unshareFeatureStoreServiceDataset(user, project, targetProject, datasetSharedWith, + Settings.ServiceDataset.DATAVALIDATION, dfso); // Unshare the online feature store try { onlineFeaturestoreController.unshareOnlineFeatureStore(targetProject, dataset.getFeatureStore()); @@ -1364,12 +1405,16 @@ public void updateSharePermission(Dataset ds, DatasetAccessPermission datasetPer PermissionTransition.valueOf(datasetSharedWith.getPermission(), datasetPermissions); updateSharePermission(datasetSharedWith, permissionTransition, project, user); - if (datasetSharedWith.getDataset().getDsType() == DatasetType.FEATURESTORE) { + //This is not needed because feature store can only be shared read only + if (DatasetType.FEATURESTORE.equals(datasetSharedWith.getDataset().getDsType())) { DatasetSharedWith trainingDataset = getSharedTrainingDataset(targetProject, datasetSharedWith.getDataset().getProject()); - if (trainingDataset != null) { - updateSharePermission(trainingDataset, permissionTransition, project, user); - } + updateSharePermissionFeatureStoreServiceDataset(user, project, permissionTransition, trainingDataset); + + updateSharePermissionFeatureStoreServiceDataset(user, project, targetProject, permissionTransition, + Settings.ServiceDataset.STATISTICS); + updateSharePermissionFeatureStoreServiceDataset(user, project, targetProject, permissionTransition, + Settings.ServiceDataset.DATAVALIDATION); // Share online feature store will revoke existing permissions onlineFeaturestoreController.shareOnlineFeatureStore(targetProject, datasetSharedWith.getDataset().getFeatureStore(), @@ -1377,6 +1422,19 @@ public void updateSharePermission(Dataset ds, DatasetAccessPermission datasetPer } } + private void updateSharePermissionFeatureStoreServiceDataset(Users user, Project project, Project targetProject, + PermissionTransition permissionTransition, Settings.ServiceDataset serviceDataset) throws DatasetException { + DatasetSharedWith datasetSharedWith = getSharedFeatureStoreServiceDataset(targetProject, project, serviceDataset); + updateSharePermissionFeatureStoreServiceDataset(user, project, permissionTransition, datasetSharedWith); + } + + private void updateSharePermissionFeatureStoreServiceDataset(Users user, Project targetProject, + PermissionTransition permissionTransition, DatasetSharedWith dataset) throws DatasetException { + if (dataset != null) { + updateSharePermission(dataset, permissionTransition, targetProject, user); + } + } + private void updateSharePermission(DatasetSharedWith datasetSharedWith, PermissionTransition permissionTransition, Project project, Users user) throws DatasetException { changePermissions(datasetSharedWith, permissionTransition); diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java index 3adc82fd65..9e5e2431ed 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java @@ -431,6 +431,9 @@ public class Settings implements Serializable { // Conda install enable flag private static final String VARIABLE_ENABLE_CONDA_INSTALL = "enable_conda_install"; + private static final String VARIABLE_FEATURE_STORE_PROJECT_ID = "default_feature_store_project_id"; + + private static final String VARIABLE_SKIP_NAMESPACE_CREATION = "kube_skip_namespace_creation"; public enum KubeType{ @@ -979,6 +982,7 @@ private void populateCache() { VARIABLE_NUM_OPENSEARCH_DEFAULT_EMBEDDING_INDEX, OPENSEARCH_NUM_DEFAULT_EMBEDDING_INDEX); ENABLE_CONDA_INSTALL = setBoolVar(VARIABLE_ENABLE_CONDA_INSTALL, ENABLE_CONDA_INSTALL); + DEFAULT_FEATURE_STORE_PROJECT_ID = setIntVar(VARIABLE_FEATURE_STORE_PROJECT_ID, null); cached = true; } } @@ -3856,4 +3860,10 @@ public synchronized boolean getEnableCondaInstall() { checkCache(); return ENABLE_CONDA_INSTALL; } + + private Integer DEFAULT_FEATURE_STORE_PROJECT_ID = null; + public synchronized Integer getDefaultFeatureStoreProjectId() { + checkCache(); + return DEFAULT_FEATURE_STORE_PROJECT_ID; + } } \ No newline at end of file