chore: unify the handling of list variable shadows (#1228)

All shadow variables related to a list variable are now handled in one
unified loop, as opposed to each having its own variable listener.
This should technically be a performance improvement, but in fact a
barely significant regression of 3 % was measured in some cases. (Other
cases show minor improvements.)

This change also makes internalized previous/next shadows possible, and
that in turn will help with the revamp of shadow variables that is
currently in development.
Therefore we introduce this change even though it may bring a small
performance regression.

core/src/main/java/ai/timefold/solver/core/impl/domain/variable/AbstractExternalizedNextPrevElementVariableProcessor.java

@@ -0,0 +1,36 @@
+package ai.timefold.solver.core.impl.domain.variable;
+
+import java.util.List;
+import java.util.Objects;
+
+import ai.timefold.solver.core.impl.domain.variable.descriptor.ShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.score.director.InnerScoreDirector;
+
+abstract sealed class AbstractExternalizedNextPrevElementVariableProcessor<Solution_>
+        permits ExternalizedNextElementVariableProcessor, ExternalizedPreviousElementVariableProcessor {
+
+    protected final ShadowVariableDescriptor<Solution_> shadowVariableDescriptor;
+
+    protected AbstractExternalizedNextPrevElementVariableProcessor(
+            ShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        this.shadowVariableDescriptor = Objects.requireNonNull(shadowVariableDescriptor);
+    }
+
+    public abstract void setElement(InnerScoreDirector<Solution_, ?> scoreDirector, List<Object> listVariable, Object element,
+            int index);
+
+    public abstract Object getElement(Object element);
+
+    public void unsetElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element) {
+        setValue(scoreDirector, element, null);
+    }
+
+    protected void setValue(InnerScoreDirector<Solution_, ?> scoreDirector, Object element, Object value) {
+        if (getElement(element) != value) {
+            scoreDirector.beforeVariableChanged(shadowVariableDescriptor, element);
+            shadowVariableDescriptor.setValue(element, value);
+            scoreDirector.afterVariableChanged(shadowVariableDescriptor, element);
+        }
+    }
+
+}

core/src/main/java/ai/timefold/solver/core/impl/domain/variable/ExternalizedIndexVariableProcessor.java

@@ -0,0 +1,45 @@
+package ai.timefold.solver.core.impl.domain.variable;
+
+import java.util.Objects;
+
+import ai.timefold.solver.core.impl.domain.variable.index.IndexShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.score.director.InnerScoreDirector;
+
+final class ExternalizedIndexVariableProcessor<Solution_> {
+
+    private final IndexShadowVariableDescriptor<Solution_> shadowVariableDescriptor;
+
+    public ExternalizedIndexVariableProcessor(IndexShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        this.shadowVariableDescriptor = shadowVariableDescriptor;
+    }
+
+    public void addElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element, Integer index) {
+        updateIndex(scoreDirector, element, index);
+    }
+
+    public void removeElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element) {
+        updateIndex(scoreDirector, element, null);
+    }
+
+    public void unassignElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element) {
+        removeElement(scoreDirector, element);
+    }
+
+    public void changeElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element, Integer index) {
+        updateIndex(scoreDirector, element, index);
+    }
+
+    private void updateIndex(InnerScoreDirector<Solution_, ?> scoreDirector, Object element, Integer index) {
+        var oldIndex = getIndex(element);
+        if (!Objects.equals(oldIndex, index)) {
+            scoreDirector.beforeVariableChanged(shadowVariableDescriptor, element);
+            shadowVariableDescriptor.setValue(element, index);
+            scoreDirector.afterVariableChanged(shadowVariableDescriptor, element);
+        }
+    }
+
+    public Integer getIndex(Object planningValue) {
+        return shadowVariableDescriptor.getValue(planningValue);
+    }
+
+}

core/src/main/java/ai/timefold/solver/core/impl/domain/variable/ExternalizedListInverseVariableProcessor.java

@@ -0,0 +1,63 @@
+package ai.timefold.solver.core.impl.domain.variable;
+
+import ai.timefold.solver.core.impl.domain.variable.descriptor.ListVariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.inverserelation.InverseRelationShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.score.director.InnerScoreDirector;
+
+final class ExternalizedListInverseVariableProcessor<Solution_> {
+
+    private final InverseRelationShadowVariableDescriptor<Solution_> shadowVariableDescriptor;
+    private final ListVariableDescriptor<Solution_> sourceVariableDescriptor;
+
+    public ExternalizedListInverseVariableProcessor(
+            InverseRelationShadowVariableDescriptor<Solution_> shadowVariableDescriptor,
+            ListVariableDescriptor<Solution_> sourceVariableDescriptor) {
+        this.shadowVariableDescriptor = shadowVariableDescriptor;
+        this.sourceVariableDescriptor = sourceVariableDescriptor;
+    }
+
+    public void addElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object entity, Object element) {
+        setInverseAsserted(scoreDirector, element, entity, null);
+    }
+
+    private void setInverseAsserted(InnerScoreDirector<Solution_, ?> scoreDirector, Object element, Object inverseEntity,
+            Object expectedOldInverseEntity) {
+        var oldInverseEntity = getInverseSingleton(element);
+        if (oldInverseEntity == inverseEntity) {
+            return;
+        }
+        if (scoreDirector.expectShadowVariablesInCorrectState() && oldInverseEntity != expectedOldInverseEntity) {
+            throw new IllegalStateException("""
+                    The entity (%s) has a list variable (%s) and one of its elements (%s) which has a shadow variable (%s) \
+                    has an oldInverseEntity (%s) which is not that entity.
+                    Verify the consistency of your input problem for that shadow variable."""
+                    .formatted(inverseEntity, sourceVariableDescriptor.getVariableName(), element,
+                            shadowVariableDescriptor.getVariableName(), oldInverseEntity));
+        }
+        setInverse(scoreDirector, inverseEntity, element);
+    }
+
+    private void setInverse(InnerScoreDirector<Solution_, ?> scoreDirector, Object entity, Object element) {
+        scoreDirector.beforeVariableChanged(shadowVariableDescriptor, element);
+        shadowVariableDescriptor.setValue(element, entity);
+        scoreDirector.afterVariableChanged(shadowVariableDescriptor, element);
+    }
+
+    public void removeElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object entity, Object element) {
+        setInverseAsserted(scoreDirector, element, null, entity);
+    }
+
+    public void unassignElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object element) {
+        changeElement(scoreDirector, null, element);
+    }
+
+    public void changeElement(InnerScoreDirector<Solution_, ?> scoreDirector, Object entity, Object element) {
+        if (getInverseSingleton(element) != entity) {
+            setInverse(scoreDirector, entity, element);
+        }
+    }
+
+    public Object getInverseSingleton(Object planningValue) {
+        return shadowVariableDescriptor.getValue(planningValue);
+    }
+}

core/src/main/java/ai/timefold/solver/core/impl/domain/variable/ExternalizedListVariableStateSupply.java

@@ -1,175 +1,161 @@
 package ai.timefold.solver.core.impl.domain.variable;
 
-import java.util.IdentityHashMap;
-import java.util.Map;
-import java.util.Objects;
-
 import ai.timefold.solver.core.api.score.director.ScoreDirector;
 import ai.timefold.solver.core.impl.domain.variable.descriptor.ListVariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.index.IndexShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.inverserelation.InverseRelationShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.nextprev.NextElementShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.nextprev.PreviousElementShadowVariableDescriptor;
+import ai.timefold.solver.core.impl.score.director.InnerScoreDirector;
 import ai.timefold.solver.core.preview.api.domain.metamodel.ElementLocation;
-import ai.timefold.solver.core.preview.api.domain.metamodel.LocationInList;
 
 import org.jspecify.annotations.NonNull;
 
 final class ExternalizedListVariableStateSupply<Solution_>
         implements ListVariableStateSupply<Solution_> {
 
     private final ListVariableDescriptor<Solution_> sourceVariableDescriptor;
-    private Map<Object, LocationInList> elementLocationMap;
-    private int unassignedCount;
+    private final ListVariableState<Solution_> listVariableState;
+
+    private boolean previousExternalized = false;
+    private boolean nextExternalized = false;
 
     public ExternalizedListVariableStateSupply(ListVariableDescriptor<Solution_> sourceVariableDescriptor) {
         this.sourceVariableDescriptor = sourceVariableDescriptor;
+        this.listVariableState = new ListVariableState<>(sourceVariableDescriptor);
+    }
+
+    @Override
+    public void externalize(IndexShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        listVariableState.linkDescriptor(shadowVariableDescriptor);
+    }
+
+    @Override
+    public void externalize(InverseRelationShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        listVariableState.linkDescriptor(shadowVariableDescriptor);
+    }
+
+    @Override
+    public void externalize(PreviousElementShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        listVariableState.linkDescriptor(shadowVariableDescriptor);
+        previousExternalized = true;
+    }
+
+    @Override
+    public void externalize(NextElementShadowVariableDescriptor<Solution_> shadowVariableDescriptor) {
+        listVariableState.linkDescriptor(shadowVariableDescriptor);
+        nextExternalized = true;
     }
 
     @Override
     public void resetWorkingSolution(@NonNull ScoreDirector<Solution_> scoreDirector) {
+        listVariableState.initialize((InnerScoreDirector<Solution_, ?>) scoreDirector,
+                (int) sourceVariableDescriptor.getValueRangeSize(scoreDirector.getWorkingSolution(), null));
         var workingSolution = scoreDirector.getWorkingSolution();
-        if (elementLocationMap == null) {
-            elementLocationMap = new IdentityHashMap<>((int) sourceVariableDescriptor.getValueRangeSize(workingSolution, null));
-        } else {
-            elementLocationMap.clear();
-        }
-        // Start with everything unassigned.
-        unassignedCount = (int) sourceVariableDescriptor.getValueRangeSize(workingSolution, null);
         // Will run over all entities and unmark all present elements as unassigned.
-        sourceVariableDescriptor.getEntityDescriptor().visitAllEntities(workingSolution, this::insert);
+        sourceVariableDescriptor.getEntityDescriptor()
+                .visitAllEntities(workingSolution, this::insert);
     }
 
     private void insert(Object entity) {
         var assignedElements = sourceVariableDescriptor.getValue(entity);
         var index = 0;
         for (var element : assignedElements) {
-            var oldLocation = elementLocationMap.put(element, ElementLocation.of(entity, index));
-            if (oldLocation != null) {
-                throw new IllegalStateException(
-                        "The supply (%s) is corrupted, because the element (%s) at index (%d) already exists (%s)."
-                                .formatted(this, element, index, oldLocation));
-            }
+            listVariableState.addElement(entity, assignedElements, element, index);
             index++;
-            unassignedCount--;
         }
     }
 
     @Override
-    public void close() {
-        elementLocationMap = null;
-    }
-
-    @Override
-    public void beforeEntityAdded(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o) {
+    public void beforeEntityAdded(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity) {
         // No need to do anything.
     }
 
     @Override
-    public void afterEntityAdded(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o) {
-        insert(o);
+    public void afterEntityAdded(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity) {
+        insert(entity);
     }
 
     @Override
-    public void beforeEntityRemoved(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o) {
+    public void beforeEntityRemoved(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity) {
         // No need to do anything.
     }
 
     @Override
-    public void afterEntityRemoved(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o) {
+    public void afterEntityRemoved(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity) {
         // When the entity is removed, its values become unassigned.
         // An unassigned value has no inverse entity and no index.
-        retract(o);
-    }
-
-    private void retract(Object entity) {
         var assignedElements = sourceVariableDescriptor.getValue(entity);
         for (var index = 0; index < assignedElements.size(); index++) {
-            var element = assignedElements.get(index);
-            var oldElementLocation = elementLocationMap.remove(element);
-            if (oldElementLocation == null) {
-                throw new IllegalStateException(
-                        "The supply (%s) is corrupted, because the element (%s) at index (%d) was already unassigned (%s)."
-                                .formatted(this, element, index, oldElementLocation));
-            }
-            var oldIndex = oldElementLocation.index();
-            if (oldIndex != index) {
-                throw new IllegalStateException(
-                        "The supply (%s) is corrupted, because the element (%s) at index (%d) had an old index (%d) which is not the current index (%d)."
-                                .formatted(this, element, index, oldIndex, index));
-            }
-            unassignedCount++;
+            listVariableState.removeElement(entity, assignedElements.get(index), index);
         }
     }
 
     @Override
     public void afterListVariableElementUnassigned(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object element) {
-        var oldLocation = elementLocationMap.remove(element);
-        if (oldLocation == null) {
-            throw new IllegalStateException(
-                    "The supply (%s) is corrupted, because the element (%s) did not exist before unassigning."
-                            .formatted(this, element));
-        }
-        unassignedCount++;
+        listVariableState.unassignElement(element);
     }
 
     @Override
-    public void beforeListVariableChanged(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o, int fromIndex,
-            int toIndex) {
+    public void beforeListVariableChanged(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity,
+            int fromIndex, int toIndex) {
         // No need to do anything.
     }
 
     @Override
-    public void afterListVariableChanged(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object o, int fromIndex,
+    public void afterListVariableChanged(@NonNull ScoreDirector<Solution_> scoreDirector, @NonNull Object entity, int fromIndex,
             int toIndex) {
-        updateIndexes(o, fromIndex, toIndex);
-    }
-
-    private void updateIndexes(Object entity, int startIndex, int toIndex) {
         var assignedElements = sourceVariableDescriptor.getValue(entity);
-        for (var index = startIndex; index < assignedElements.size(); index++) {
-            var element = assignedElements.get(index);
-            var newLocation = ElementLocation.of(entity, index);
-            var oldLocation = elementLocationMap.put(element, newLocation);
-            if (oldLocation == null) {
-                unassignedCount--;
-            } else if (index >= toIndex && newLocation.equals(oldLocation)) {
+        var elementCount = assignedElements.size();
+        // Include the last element of the previous part of the list, if any, for the next element shadow var.
+        // But only if the next element shadow var is externalized; otherwise, there is nothing to update.
+        var firstChangeIndex = nextExternalized ? Math.max(0, fromIndex - 1) : fromIndex;
+        // Include the first element of the next part of the list, if any, for the previous element shadow var.
+        // But only if the previous element shadow var is externalized; otherwise, there is nothing to update.
+        var lastChangeIndex = previousExternalized ? Math.min(toIndex + 1, elementCount) : toIndex;
+        for (var index = firstChangeIndex; index < elementCount; index++) {
+            var locationsDiffer = listVariableState.changeElement(entity, assignedElements, index);
+            if (!locationsDiffer && index >= lastChangeIndex) {
                 // Location is unchanged and we are past the part of the list that changed.
+                // We can terminate the loop prematurely.
                 return;
-            } else {
-                // Continue to the next element.
             }
         }
     }
 
     @Override
     public ElementLocation getLocationInList(Object planningValue) {
-        return Objects.requireNonNullElse(elementLocationMap.get(Objects.requireNonNull(planningValue)),
-                ElementLocation.unassigned());
+        return listVariableState.getLocationInList(planningValue);
     }
 
     @Override
     public Integer getIndex(Object planningValue) {
-        var elementLocation = elementLocationMap.get(Objects.requireNonNull(planningValue));
-        if (elementLocation == null) {
-            return null;
-        }
-        return elementLocation.index();
+        return listVariableState.getIndex(planningValue);
     }
 
     @Override
     public Object getInverseSingleton(Object planningValue) {
-        var elementLocation = elementLocationMap.get(Objects.requireNonNull(planningValue));
-        if (elementLocation == null) {
-            return null;
-        }
-        return elementLocation.entity();
+        return listVariableState.getInverseSingleton(planningValue);
     }
 
     @Override
     public boolean isAssigned(Object element) {
-        return getLocationInList(element) instanceof LocationInList;
+        return getInverseSingleton(element) != null;
     }
 
     @Override
     public int getUnassignedCount() {
-        return unassignedCount;
+        return listVariableState.getUnassignedCount();
+    }
+
+    @Override
+    public Object getPreviousElement(Object element) {
+        return listVariableState.getPreviousElement(element);
+    }
+
+    @Override
+    public Object getNextElement(Object element) {
+        return listVariableState.getNextElement(element);
     }
 
     @Override