Skip to content

Commit

Permalink
MNT-24503 - Limits on FixedAclUpdater (#2788)
Browse files Browse the repository at this point in the history 
    * Limit the job execution to a configurable amount of nodes (system.fixedACLsUpdater.maxItems)
    * Add query template select_NodesWithAspectIds_Limited that does a left join with alf_store and limits the results
    * Query executions by the job are now limited to maxItemBatchSize
    * Imposing the order by in the query is now configurable (system.fixedACLsUpdater.orderNodes)
  • Loading branch information
@evasques
evasques authored Jul 17, 2024
1 parent b63a3ea commit 9ed2996
Show file tree
Hide file tree
Showing 9 changed files with 251 additions and 12 deletions.
10 changes: 10 additions & 0 deletions repository/src/main/java/org/alfresco/ibatis/IdsEntity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public class IdsEntity
private Long idFour;
private List<Long> ids;
private boolean ordered;
private Integer maxResults;

public Long getIdOne()
{
return idOne;
Expand Down Expand Up @@ -89,4 +91,12 @@ public void setOrdered(boolean ordered)
{
this.ordered = ordered;
}
public int getMaxResults()
{
return maxResults;
}
public void setMaxResults(Integer maxResults)
{
this.maxResults = maxResults;
}
}
22 changes: 22 additions & 0 deletions repository/src/main/java/org/alfresco/repo/domain/node/AbstractNodeDAOImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2785,6 +2785,23 @@ public void getNodesWithAspects(
selectNodesWithAspects(qnameIds, minNodeId, maxNodeId, ordered, resultsCallback);
}

@Override
public void getNodesWithAspects(
Set<QName> aspectQNames,
Long minNodeId, Long maxNodeId, boolean ordered,
int maxResults,
NodeRefQueryCallback resultsCallback)
{
Set<Long> qnameIdsSet = qnameDAO.convertQNamesToIds(aspectQNames, false);
if (qnameIdsSet.isEmpty())
{
// No point running a query
return;
}
List<Long> qnameIds = new ArrayList<>(qnameIdsSet);
selectNodesWithAspects(qnameIds, minNodeId, maxNodeId, ordered, maxResults, resultsCallback);
}

/**
* @return Returns a writable copy of the cached aspects set
*/
Expand Down Expand Up @@ -4960,6 +4977,10 @@ protected abstract void selectNodesWithAspects(
List<Long> qnameIds,
Long minNodeId, Long maxNodeId, boolean ordered,
NodeRefQueryCallback resultsCallback);
protected abstract void selectNodesWithAspects(
List<Long> qnameIds,
Long minNodeId, Long maxNodeId, boolean ordered, int maxResults,
NodeRefQueryCallback resultsCallback);
protected abstract Long insertNodeAssoc(Long sourceNodeId, Long targetNodeId, Long assocTypeQNameId, int assocIndex);
protected abstract int updateNodeAssoc(Long id, int assocIndex);
protected abstract int deleteNodeAssoc(Long sourceNodeId, Long targetNodeId, Long assocTypeQNameId);
Expand Down Expand Up @@ -5088,4 +5109,5 @@ public abstract List<Transaction> selectTxns(
protected abstract Long selectMinTxInNodeIdRange(Long fromNodeId, Long toNodeId);
protected abstract Long selectMaxTxInNodeIdRange(Long fromNodeId, Long toNodeId);
protected abstract Long selectNextTxCommitTime(Long fromCommitTime);

}
16 changes: 16 additions & 0 deletions repository/src/main/java/org/alfresco/repo/domain/node/NodeDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,22 @@ public void getNodesWithAspects(
Long minNodeId, Long maxNodeId, boolean ordered,
NodeRefQueryCallback resultsCallback);

/**
* Get nodes with aspects between the given ranges, ordering the results optionally
* and limit the result set
*
* @param aspectQNames the aspects that must be on the nodes
* @param minNodeId the minimum node ID (inclusive)
* @param maxNodeId the maximum node ID (exclusive)
* @param ordered if the results are to be ordered by nodeID
* @param maxResults limit query to maxResults
* @param resultsCallback callback to process results
*/
public void getNodesWithAspects(
Set<QName> aspectQNames,
Long minNodeId, Long maxNodeId, boolean ordered, int maxResults,
NodeRefQueryCallback resultsCallback);

/*
* Node Assocs
*/
Expand Down
28 changes: 28 additions & 0 deletions repository/src/main/java/org/alfresco/repo/domain/node/ibatis/NodeDAOImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public class NodeDAOImpl extends AbstractNodeDAOImpl
private static final String SELECT_NODE_MAX_ID = "alfresco.node.select_NodeMaxId";
private static final String SELECT_NODE_INTERVAL_BY_TYPE = "alfresco.node.select_MinMaxNodeIdForNodeType";
private static final String SELECT_NODES_WITH_ASPECT_IDS = "alfresco.node.select_NodesWithAspectIds";
private static final String SELECT_NODES_WITH_ASPECT_IDS_LIMITED = "alfresco.node.select_NodesWithAspectIds_Limited";
private static final String INSERT_NODE_ASSOC = "alfresco.node.insert.insert_NodeAssoc";
private static final String UPDATE_NODE_ASSOC = "alfresco.node.update_NodeAssoc";
private static final String DELETE_NODE_ASSOC = "alfresco.node.delete_NodeAssoc";
Expand Down Expand Up @@ -799,6 +800,33 @@ public void handleResult(ResultContext context)
template.select(SELECT_NODES_WITH_ASPECT_IDS, parameters, resultHandler);
}

@Override
protected void selectNodesWithAspects(
List<Long> qnameIds,
Long minNodeId, Long maxNodeId, boolean ordered,
final int maxResults,
final NodeRefQueryCallback resultsCallback)
{
@SuppressWarnings("rawtypes")
ResultHandler resultHandler = new ResultHandler()
{
public void handleResult(ResultContext context)
{
NodeEntity entity = (NodeEntity) context.getResultObject();
Pair<Long, NodeRef> nodePair = new Pair<>(entity.getId(), entity.getNodeRef());
resultsCallback.handle(nodePair);
}
};

IdsEntity parameters = new IdsEntity();
parameters.setIdOne(minNodeId);
parameters.setIdTwo(maxNodeId);
parameters.setIds(qnameIds);
parameters.setOrdered(ordered);
parameters.setMaxResults(maxResults);
template.select(SELECT_NODES_WITH_ASPECT_IDS_LIMITED, parameters, resultHandler);
}

@Override
protected Long insertNodeAssoc(Long sourceNodeId, Long targetNodeId, Long assocTypeQNameId, int assocIndex)
{
Expand Down
59 changes: 57 additions & 2 deletions repository/src/main/java/org/alfresco/repo/domain/permissions/FixedAclUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,11 @@ public class FixedAclUpdater extends TransactionListenerAdapter implements Appli

public static final String FIXED_ACL_ASYNC_REQUIRED_KEY = "FIXED_ACL_ASYNC_REQUIRED";
public static final String FIXED_ACL_ASYNC_CALL_KEY = "FIXED_ACL_ASYNC_CALL";

protected static final QName LOCK_Q_NAME = QName.createQName(NamespaceService.SYSTEM_MODEL_1_0_URI, "FixedAclUpdater");

private static final int DEFAULT_MAX_ITEMS = Integer.MAX_VALUE;

/** A set of listeners to receive callback events whenever permissions are updated by this class. */
private static Set<FixedAclUpdaterListener> listeners = Sets.newConcurrentHashSet();

Expand All @@ -101,6 +104,8 @@ public class FixedAclUpdater extends TransactionListenerAdapter implements Appli
private int maxItemBatchSize = 100;
private int numThreads = 4;
private boolean forceSharedACL = false;
private int maxItems = DEFAULT_MAX_ITEMS;
private boolean orderNodes = true;

private ClassPolicyDelegate<OnInheritPermissionsDisabled> onInheritPermissionsDisabledDelegate;
private PolicyComponent policyComponent;
Expand Down Expand Up @@ -147,12 +152,22 @@ public void setForceSharedACL(boolean forceSharedACL)
this.forceSharedACL = forceSharedACL;
}

public void setOrderNodes(boolean orderNodes)
{
this.orderNodes = orderNodes;
}

public void setLockTimeToLive(long lockTimeToLive)
{
this.lockTimeToLive = lockTimeToLive;
this.lockRefreshTime = lockTimeToLive / 2;
}

public void setMaxItems(int maxItems)
{
this.maxItems = maxItems > 0 ? maxItems : DEFAULT_MAX_ITEMS;
}

public void setPolicyComponent(PolicyComponent policyComponent)
{
this.policyComponent = policyComponent;
Expand Down Expand Up @@ -209,7 +224,7 @@ List<NodeRef> getNodesWithAspects()
public List<NodeRef> execute() throws Throwable
{
getNodesCallback.init();
nodeDAO.getNodesWithAspects(aspects, getNodesCallback.getMinNodeId(), null, true, getNodesCallback);
nodeDAO.getNodesWithAspects(aspects, getNodesCallback.getMinNodeId(), null, orderNodes, maxItemBatchSize, getNodesCallback);
getNodesCallback.done();

return getNodesCallback.getNodes();
Expand All @@ -231,13 +246,22 @@ public Integer execute() throws Throwable
return countNodesCallback.getCount();
}
}, false, true);

if (count > maxItems)
{
log.info("Total nodes with pending acl: " + count + " Limiting work to " + maxItems);
return maxItems;
}
return count;
}
}

private class AclWorkProvider implements BatchProcessWorkProvider<NodeRef>
{
private GetNodesWithAspects getNodesWithAspects;
private long estimatedUpdatedItems;
private long execTime;
private long execBatches;

AclWorkProvider()
{
Expand All @@ -259,8 +283,37 @@ public long getTotalEstimatedWorkSizeLong()
@Override
public Collection<NodeRef> getNextWork()
{
return getNodesWithAspects.getNodesWithAspects();
if(estimatedUpdatedItems >= maxItems)
{
log.info("Reached max items to process. Nodes Processed: " + estimatedUpdatedItems + "/" + maxItems);
return Collections.emptyList();
}

long initTime = System.currentTimeMillis();
Collection<NodeRef> batchNodes = getNodesWithAspects.getNodesWithAspects();
long endTime = System.currentTimeMillis();

if (log.isDebugEnabled())
{
log.debug("Query for batch executed in " + (endTime-initTime) + " ms");
}

if (!batchNodes.isEmpty())
{
// Increment estimatedUpdatedItems with the expected number of nodes to process
estimatedUpdatedItems += batchNodes.size();
execTime+=endTime-initTime;
execBatches++;
}

return batchNodes;
}

public double getAverageQueryExecutionTime()
{
return execBatches > 0 ? execTime/execBatches : 0;
}

}

protected class AclWorker implements BatchProcessor.BatchProcessWorker<NodeRef>
Expand Down Expand Up @@ -451,6 +504,7 @@ public int execute()

try
{
log.info("Running FixedAclUpdater. Max Items: " + maxItems + ", Impose order: " + orderNodes);
lockToken = jobLockService.getLock(LOCK_Q_NAME, lockTimeToLive, 0, 1);
jobLockService.refreshLock(lockToken, LOCK_Q_NAME, lockRefreshTime, jobLockRefreshCallback);

Expand All @@ -460,6 +514,7 @@ public int execute()
transactionService.getRetryingTransactionHelper(), provider, numThreads, maxItemBatchSize, applicationContext,
log, 100);
int count = bp.process(worker, true);
log.info("FixedAclUpdater updated " + count + ". Average query time " + provider.getAverageQueryExecutionTime() + " ms");
return count;
}
catch (LockAcquisitionException e)
Expand Down
19 changes: 19 additions & 0 deletions ...resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/node-common-SqlMap.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,25 @@
<if test="ordered == true">order by node.id ASC</if>
</select>

<select id="select_NodesWithAspectIds_Limited" parameterType="Ids" resultMap="result_NodeRef" >
select
node.id as id,
store.protocol as protocol,
store.identifier as identifier,
node.uuid as uuid
from
alf_node_aspects na
join alf_node node on (na.node_id = node.id)
left join alf_store store on (store.id = node.store_id)
where
<![CDATA[na.node_id >= #{idOne}]]>
<if test="idTwo != null"><![CDATA[and na.node_id < #{idTwo}]]></if>
and na.qname_id in
<foreach item="item" index="i" collection="ids" open="(" separator="," close=")">#{item}</foreach>
<if test="ordered == true">order by node.id ASC</if>
<if test="maxResults != null"><![CDATA[limit #{maxResults}]]></if>
</select>

<!-- Common results for result_NodeAssoc -->
<sql id="select_NodeAssoc_Results">
select
Expand Down
2 changes: 2 additions & 0 deletions repository/src/main/resources/alfresco/public-services-security-context.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@
<property name="maxItemBatchSize" value="${system.fixedACLsUpdater.maxItemBatchSize}"/>
<property name="numThreads" value="${system.fixedACLsUpdater.numThreads}"/>
<property name="forceSharedACL" value="${system.fixedACLsUpdater.forceSharedACL}"/>
<property name="maxItems" value="${system.fixedACLsUpdater.maxItems}"/>
<property name="orderNodes" value="${system.fixedACLsUpdater.orderNodes}"/>
<property name="lockTimeToLive" value="${system.fixedACLsUpdater.lockTTL}"/>
<property name="policyComponent" ref="policyComponent"/>
<property name="policyIgnoreUtil" ref="policyIgnoreUtil"/>
Expand Down
6 changes: 5 additions & 1 deletion repository/src/main/resources/alfresco/repository.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,7 +1104,11 @@ system.fixedACLsUpdater.numThreads=4
# fixedACLsUpdater - Force shared ACL to propagate through children even if there is an unexpected ACL
system.fixedACLsUpdater.forceSharedACL=false
# fixedACLsUpdater cron expression - fire at midnight every day
system.fixedACLsUpdater.cronExpression=0 0 0 * * ?
system.fixedACLsUpdater.cronExpression=0 0 0 * * ?
# fixedACLsUpdater - maximum number of pending ACLs to process overall
system.fixedACLsUpdater.maxItems=-1
# fixedACLsUpdater - Impose the order by in the query. If false, it may not process all the results but should do the queries faster
system.fixedACLsUpdater.orderNodes=true

cmis.disable.hidden.leading.period.files=false

Expand Down
Loading

0 comments on commit 9ed2996

Please sign in to comment.