From 846c3e1c3baa53e5355ab28ef535c61a5ae5bd00 Mon Sep 17 00:00:00 2001 From: Yang Tang <45587763+tangyang9464@users.noreply.github.com> Date: Wed, 6 Apr 2022 19:48:30 +0800 Subject: [PATCH] feat: implement batch adapter (#47) Signed-off-by: tangyang9464 --- pom.xml | 14 ++- .../java/org/casbin/adapter/JDBCAdapter.java | 5 +- .../org/casbin/adapter/JDBCBaseAdapter.java | 93 ++++++++++++++----- .../org/casbin/adapter/AdapterCreator.java | 2 +- .../casbin/adapter/JDBCAdapterTestSets.java | 8 ++ 5 files changed, 91 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 4b7b3a2..e8a6bbb 100644 --- a/pom.xml +++ b/pom.xml @@ -182,21 +182,27 @@ + + org.apache.commons + commons-collections4 + 4.4 + + junit junit - 4.13.1 + 4.13.2 test org.casbin jcasbin - 1.6.2 + 1.22.2 mysql mysql-connector-java - 8.0.16 + 8.0.28 test @@ -220,7 +226,7 @@ dev.failsafe failsafe - 3.0.0 + 3.2.3 diff --git a/src/main/java/org/casbin/adapter/JDBCAdapter.java b/src/main/java/org/casbin/adapter/JDBCAdapter.java index a0a42dd..b234a1a 100644 --- a/src/main/java/org/casbin/adapter/JDBCAdapter.java +++ b/src/main/java/org/casbin/adapter/JDBCAdapter.java @@ -22,10 +22,7 @@ import org.casbin.jcasbin.persist.file_adapter.FilteredAdapter.Filter; import javax.sql.DataSource; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; /** * JDBCFilteredAdapter is the JDBC adapter for jCasbin. diff --git a/src/main/java/org/casbin/adapter/JDBCBaseAdapter.java b/src/main/java/org/casbin/adapter/JDBCBaseAdapter.java index dda6ef9..2e50ed3 100644 --- a/src/main/java/org/casbin/adapter/JDBCBaseAdapter.java +++ b/src/main/java/org/casbin/adapter/JDBCBaseAdapter.java @@ -17,10 +17,11 @@ import dev.failsafe.ExecutionContext; import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.casbin.jcasbin.model.Assertion; import org.casbin.jcasbin.model.Model; import org.casbin.jcasbin.persist.Adapter; +import org.casbin.jcasbin.persist.BatchAdapter; import org.casbin.jcasbin.persist.Helper; import javax.sql.DataSource; @@ -47,7 +48,7 @@ public String[] toStringArray() { * JDBCAdapter is the JDBC adapter for jCasbin. * It can load policy from JDBC supported database or save policy to it. */ -abstract class JDBCBaseAdapter implements Adapter { +abstract class JDBCBaseAdapter implements Adapter, BatchAdapter { protected static final int _DEFAULT_CONNECTION_TRIES = 3; protected DataSource dataSource; private final int batchSize = 1000; @@ -270,8 +271,10 @@ public void savePolicy(Model model) { ps.setString(7, line.v5); ps.addBatch(); - if (++count % batchSize == 0) { + if (++count == batchSize) { + count = 0; ps.executeBatch(); + ps.clearBatch(); } } } @@ -292,13 +295,17 @@ public void savePolicy(Model model) { ps.setString(7, line.v5); ps.addBatch(); - if (++count % batchSize == 0) { + if (++count == batchSize) { + count = 0; ps.executeBatch(); + ps.clearBatch(); } } } - ps.executeBatch(); + if(count!=0){ + ps.executeBatch(); + } conn.commit(); } catch (SQLException e) { @@ -311,33 +318,54 @@ public void savePolicy(Model model) { } }); } - /** * addPolicy adds a policy rule to the storage. */ @Override public void addPolicy(String sec, String ptype, List rule) { - if (CollectionUtils.isEmpty(rule)) return; + List> rules = new ArrayList<>(); + rules.add(rule); + this.addPolicies(sec,ptype,rules); + } + + @Override + public void addPolicies(String sec, String ptype, List> rules) { + if (CollectionUtils.isEmpty(rules)) { + return; + } String sql = "INSERT INTO casbin_rule (ptype,v0,v1,v2,v3,v4,v5) VALUES(?,?,?,?,?,?,?)"; + Failsafe.with(retryPolicy).run(ctx -> { if (ctx.isRetry()) { retry(ctx); } + conn.setAutoCommit(false); + int count = 0; try (PreparedStatement ps = conn.prepareStatement(sql)) { - CasbinRule line = savePolicyLine(ptype, rule); - - ps.setString(1, line.ptype); - ps.setString(2, line.v0); - ps.setString(3, line.v1); - ps.setString(4, line.v2); - ps.setString(5, line.v3); - ps.setString(6, line.v4); - ps.setString(7, line.v5); - ps.addBatch(); - ps.executeBatch(); + for(List rule:rules){ + CasbinRule line = savePolicyLine(ptype, rule); + + ps.setString(1, line.ptype); + ps.setString(2, line.v0); + ps.setString(3, line.v1); + ps.setString(4, line.v2); + ps.setString(5, line.v3); + ps.setString(6, line.v4); + ps.setString(7, line.v5); + ps.addBatch(); + if (++count == batchSize) { + count=0; + ps.executeBatch(); + ps.clearBatch(); + } + } + if(count!=0){ + ps.executeBatch(); + } } + conn.commit(); }); } @@ -346,17 +374,39 @@ public void addPolicy(String sec, String ptype, List rule) { */ @Override public void removePolicy(String sec, String ptype, List rule) { - if (CollectionUtils.isEmpty(rule)) return; + if (CollectionUtils.isEmpty(rule)) { + return; + } removeFilteredPolicy(sec, ptype, 0, rule.toArray(new String[0])); } + @Override + public void removePolicies(String sec, String ptype, List> rules) { + if (CollectionUtils.isEmpty(rules)) { + return; + } + + Failsafe.with(retryPolicy).run(ctx -> { + if (ctx.isRetry()) { + retry(ctx); + } + conn.setAutoCommit(false); + for(List rule:rules){ + removeFilteredPolicy(sec, ptype, 0, rule.toArray(new String[0])); + } + conn.commit(); + }); + } + /** * removeFilteredPolicy removes policy rules that match the filter from the storage. */ @Override public void removeFilteredPolicy(String sec, String ptype, int fieldIndex, String... fieldValues) { List values = Optional.of(Arrays.asList(fieldValues)).orElse(new ArrayList<>()); - if (CollectionUtils.isEmpty(values)) return; + if (CollectionUtils.isEmpty(values)) { + return; + } Failsafe.with(retryPolicy).run(ctx -> { if (ctx.isRetry()) { @@ -373,8 +423,7 @@ public void removeFilteredPolicy(String sec, String ptype, int fieldIndex, Strin for (int j = 0; j < values.size(); j++) { ps.setString(j + 2, values.get(j)); } - ps.addBatch(); - ps.executeBatch(); + ps.executeUpdate(); } }); } diff --git a/src/test/java/org/casbin/adapter/AdapterCreator.java b/src/test/java/org/casbin/adapter/AdapterCreator.java index 011f027..fc69e8b 100644 --- a/src/test/java/org/casbin/adapter/AdapterCreator.java +++ b/src/test/java/org/casbin/adapter/AdapterCreator.java @@ -26,7 +26,7 @@ public interface AdapterCreator { } class MySQLAdapterCreator implements AdapterCreator { - private String url = "jdbc:mysql://127.0.0.1:3306/casbin?serverTimezone=GMT%2B8&useSSL=false&allowPublicKeyRetrieval=true"; + private String url = "jdbc:mysql://127.0.0.1:3306/casbin?serverTimezone=GMT%2B8&useSSL=false&allowPublicKeyRetrieval=true&rewriteBatchedStatements=true"; private String username = "casbin_test"; private String password = "TEST_casbin"; private String driver = "com.mysql.cj.jdbc.Driver"; diff --git a/src/test/java/org/casbin/adapter/JDBCAdapterTestSets.java b/src/test/java/org/casbin/adapter/JDBCAdapterTestSets.java index 688baed..51524ca 100644 --- a/src/test/java/org/casbin/adapter/JDBCAdapterTestSets.java +++ b/src/test/java/org/casbin/adapter/JDBCAdapterTestSets.java @@ -92,6 +92,10 @@ static void testAddAndRemovePolicy(JDBCAdapter a) { e.addPolicy("cathy", "data1", "read"); testEnforce(e, "cathy", "data1", "read", true); + e.addPolicies(new String[][]{{"cathy2", "data1", "read"},{"cathy3", "data1", "read"}}); + testEnforce(e, "cathy2", "data1", "read", true); + testEnforce(e, "cathy3", "data1", "read", true); + // Reload the policy from the storage to see the effect. e.clearPolicy(); a.loadPolicy(e.getModel()); @@ -102,6 +106,10 @@ static void testAddAndRemovePolicy(JDBCAdapter a) { e.removePolicy("cathy", "data1", "read"); testEnforce(e, "cathy", "data1", "read", false); + e.removePolicies(new String[][]{{"cathy2", "data1", "read"},{"cathy3", "data1", "read"}}); + testEnforce(e, "cathy2", "data1", "read", false); + testEnforce(e, "cathy3", "data1", "read", false); + // Reload the policy from the storage to see the effect. e.clearPolicy(); a.loadPolicy(e.getModel());