diff --git a/CHANGELOG.md b/CHANGELOG.md
index 84566f60..da68e650 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 
 ## [unreleased]
 
+✨ **layer_seq:** ValueCausalSeq ([126](https://github.com/owkin/GrAIdient/pull/126))\
 ✨ **layer_seq:** QueryCausalSeq ([125](https://github.com/owkin/GrAIdient/pull/125))\
 ✨ **layer_seq:** RoPESeq ([124](https://github.com/owkin/GrAIdient/pull/124))\
 ✨ **layer_seq:** RMSNormSeq ([123](https://github.com/owkin/GrAIdient/pull/123))\
diff --git a/Sources/GrAIdient/LayerSeq/QuerySeq.swift b/Sources/GrAIdient/LayerSeq/QuerySeq.swift
index 012fae53..31148ce1 100644
--- a/Sources/GrAIdient/LayerSeq/QuerySeq.swift
+++ b/Sources/GrAIdient/LayerSeq/QuerySeq.swift
@@ -1236,20 +1236,20 @@ public class QueryCausalSeq: LayerMergeSeq
         
         let query = (_layersPrev[0] as! LayerSeq).neurons!
         let key = (_layersPrev[1] as! LayerSeq).neurons!
+        
         let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsQuery
+        let nbBlocksHead = _nbHeadsQuery / _nbHeadsKey
         
         for batch in 0..<batchSize {
         for headQuery in 0..<_nbHeadsQuery {
+        let headKey = headQuery / nbBlocksHead
         for seqQ in 0..<sequence {
         for seqK in 0..<sequence {
         for elem in 0..<nbSameElems
         {
             if seqK <= seqQ
             {
-                let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                    headQuery : headQuery / _nbHeadsKey
                 var sum = 0.0
-                
                 for j in 0..<size
                 {
                     let depthPrevKey = j + headKey * size
@@ -1279,6 +1279,7 @@ public class QueryCausalSeq: LayerMergeSeq
         
         for batch in 0..<batchSize {
         for headQuery in 0..<_nbHeadsQuery {
+        let headKey = headQuery / nbBlocksHead
         for seqQ in 0..<sequence {
         for seqK in 0..<sequence {
         var offset = nbSameElems
@@ -1289,10 +1290,7 @@ public class QueryCausalSeq: LayerMergeSeq
         {
             if seqK <= seqQ
             {
-                let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                    headQuery : headQuery / _nbHeadsKey
                 var sum = 0.0
-                
                 for j in 0..<size
                 {
                     let depthPrevKey = j + headKey * size
@@ -1361,22 +1359,23 @@ public class QueryCausalSeq: LayerMergeSeq
         
         let query = (_layersPrev[0] as! LayerSeq).neurons!
         let key = (_layersPrev[1] as! LayerSeq).neurons!
+        
         let nbNeuronsPrevQuery = (_layersPrev[0] as! LayerSeq).nbNeurons
         let nbNeuronsPrevKey = (_layersPrev[1] as! LayerSeq).nbNeurons
+        
         let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsQuery
+        let nbBlocksHead = _nbHeadsQuery / _nbHeadsKey
         
         for batch in 0..<batchSize {
         for headQuery in 0..<_nbHeadsQuery {
+        let headKey = headQuery / nbBlocksHead
         for seqQ in 0..<sequence {
         for seqK in 0..<sequence {
         for elem in 0..<nbSameElems
         {
             if seqK <= seqQ
             {
-                let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                    headQuery : headQuery / _nbHeadsKey
                 var sum = 0.0
-                
                 for j in 0..<size
                 {
                     let depthPrevKey = j + headKey * size
@@ -1409,6 +1408,7 @@ public class QueryCausalSeq: LayerMergeSeq
         
         for batch in 0..<batchSize {
         for headQuery in 0..<_nbHeadsQuery {
+        let headKey = headQuery / nbBlocksHead
         for seqQ in 0..<sequence {
         for seqK in 0..<sequence {
         var offset = nbSameElems
@@ -1419,10 +1419,7 @@ public class QueryCausalSeq: LayerMergeSeq
         {
             if seqK <= seqQ
             {
-                let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                    headQuery : headQuery / _nbHeadsKey
                 var sum = 0.0
-                
                 for j in 0..<size
                 {
                     let depthPrevKey = j + headKey * size
@@ -1487,17 +1484,17 @@ public class QueryCausalSeq: LayerMergeSeq
         
         let query = (_layersPrev[0] as! LayerSeq).neurons!
         let key = (_layersPrev[1] as! LayerSeq).neurons!
+        
         let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsQuery
+        let nbBlocksHead = _nbHeadsQuery / _nbHeadsKey
         
         for elem in 0..<batchSize {
         for headQuery in 0..<_nbHeadsQuery {
+        let headKey = headQuery / nbBlocksHead
         for seqQ in 0..<sequence {
         for seqK in 0...seqQ
         {
-            let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                headQuery : headQuery / _nbHeadsKey
             var sum = 0.0
-            
             for j in 0..<size
             {
                 let depthPrevKey = j + headKey * size
@@ -1569,14 +1566,15 @@ public class QueryCausalSeq: LayerMergeSeq
         
         let query = (_layersPrev[0] as! LayerSeq).neurons!
         let key = (_layersPrev[1] as! LayerSeq).neurons!
+        
         let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsQuery
+        let nbBlocksHead = _nbHeadsQuery / _nbHeadsKey
         
         if _layersPrev[0].computeDelta
         {
             for elem in 0..<batchSize {
             for headQuery in 0..<_nbHeadsQuery {
-            let headKey = _nbHeadsQuery == _nbHeadsKey ?
-                headQuery : headQuery / _nbHeadsKey
+                let headKey = headQuery / nbBlocksHead
             for seqQ in 0..<sequence {
             for j in 0..<size
             {
@@ -1607,9 +1605,6 @@ public class QueryCausalSeq: LayerMergeSeq
         }
         if _layersPrev[1].computeDelta
         {
-            let nbBlocksHead = _nbHeadsQuery == _nbHeadsKey ?
-                1 : _nbHeadsQuery / _nbHeadsKey
-            
             for elem in 0..<batchSize {
             for headKey in 0..<_nbHeadsKey {
             for seqK in 0..<sequence {
diff --git a/Sources/GrAIdient/LayerSeq/ValueSeq.swift b/Sources/GrAIdient/LayerSeq/ValueSeq.swift
index 2507e484..4f0a2f7d 100644
--- a/Sources/GrAIdient/LayerSeq/ValueSeq.swift
+++ b/Sources/GrAIdient/LayerSeq/ValueSeq.swift
@@ -1152,3 +1152,607 @@ public class ValueSelfSeq: LayerMergeSeq
         propagateDirty()
     }
 }
+
+///
+/// Layer with a sequential shape neural structure.
+///
+/// This layer computes the values (value layer) that are scaled through causal attention scores (score layer).
+///
+public class ValueCausalSeq: LayerMergeSeq
+{
+    /// Number of heads (groups) of neurons for score.
+    let _nbHeadsScore: Int
+    /// Number of heads (groups) of neurons for value.
+    let _nbHeadsValue: Int
+    
+    private enum Keys: String, CodingKey
+    {
+        case nbHeadsScore
+        case nbHeadsValue
+    }
+    
+    ///
+    /// Create a layer with a sequential shape neural structure.
+    ///
+    /// - Parameters:
+    ///     - value: Previous layer containing the value.
+    ///     - score: Previous layer contianing the attention scores per sequence.
+    ///     - nbHeadsValue: Number of heads (groups) of neurons for value.
+    ///     - nbHeadsScore: Number of heads (groups) of neurons for score.
+    ///     - params: Contextual parameters linking to the model.
+    ///
+    public init(value: LayerSeq, score: LayerSeq,
+                nbHeadsValue: Int, nbHeadsScore: Int,
+                params: GrAI.Model.Params) throws
+    {
+        if value.nbNeurons % nbHeadsValue != 0
+        {
+            throw LayerError.Init(message:
+                "`nbNeurons` (\(value.nbNeurons)) " +
+                "should be a multiple of `nbHeadsValue` (\(nbHeadsValue))."
+            )
+        }
+        if score.nbNeurons % nbHeadsScore != 0
+        {
+            throw LayerError.Init(message:
+                "`nbNeurons` (\(score.nbNeurons)) " +
+                "should be a multiple of `nbHeadsScore` (\(nbHeadsScore))."
+            )
+        }
+        if nbHeadsScore % nbHeadsValue != 0
+        {
+            throw LayerError.Init(message:
+                "`nbHeadsScore` should be a multiple of `nbHeadsValue`"
+            )
+        }
+        if value.sequence != score.sequence
+        {
+            throw LayerError.Init(message: "Layer structure error.")
+        }
+
+        _nbHeadsScore = nbHeadsScore
+        _nbHeadsValue = nbHeadsValue
+        
+        super.init(layersPrev: [value, score],
+                   sequence: value.sequence,
+                   nbNeurons: (value.nbNeurons / nbHeadsValue) * nbHeadsScore,
+                   params: params)
+    }
+    
+    ///
+    /// Decode from the disk.
+    ///
+    /// Throw an error if reading from the decoder fails, or
+    /// if the data read is corrupted or otherwise invalid.
+    ///
+    /// - Parameter decoder: The decoder to read data from.
+    ///
+    public required init(from decoder: Decoder) throws
+    {
+        let values = try decoder.container(keyedBy: Keys.self)
+        _nbHeadsScore = try values.decode(Int.self, forKey: Keys.nbHeadsScore)
+        _nbHeadsValue = try values.decode(Int.self, forKey: Keys.nbHeadsValue)
+        try super.init(from: decoder)
+    }
+    
+    ///
+    /// Encode to the disk.
+    ///
+    /// If the value fails to encode anything, `encoder` will encode an empty
+    /// keyed container in its place.
+    ///
+    /// Throw an error if any values are invalid for the given
+    /// encoder's format.
+    ///
+    /// - Parameter encoder: The encoder to write data to.
+    ///
+    public override func encode(to encoder: Encoder) throws
+    {
+        var container = encoder.container(keyedBy: Keys.self)
+        try container.encode(_nbHeadsScore, forKey: Keys.nbHeadsScore)
+        try container.encode(_nbHeadsValue, forKey: Keys.nbHeadsValue)
+        try super.encode(to: encoder)
+    }
+    
+    ///
+    /// Create a layer with same values as this.
+    ///
+    /// - Parameters:
+    ///     - mapping: Dictionary allowing to find the layer associated to some id.
+    ///     This dictionary is particularly useful when the different layers cannot access
+    ///     their `layerPrev`.
+    ///     - inPlace: Whether hard resources should be copied as is.
+    ///
+    /// - Returns: A new layer. When `inPlace` is false, `initKernel` is
+    /// necessary in order to recreate hard resources.
+    ///
+    public override func copy(
+        mapping: Dictionary<Int, Layer>,
+        inPlace: Bool) -> Layer
+    {
+        let context = ModelContext(name: "", curID: 0)
+        let params = GrAI.Model.Params(context: context)
+        params.context.curID = id
+        
+        var layersPrev = [LayerSeq]()
+        for idPrev in _idsPrev
+        {
+            layersPrev.append(mapping[idPrev] as! LayerSeq)
+        }
+        
+        let layer = try! ValueCausalSeq(
+            value: layersPrev[0], score: layersPrev[1],
+            nbHeadsValue: _nbHeadsValue,
+            nbHeadsScore: _nbHeadsScore,
+            params: params
+        )
+        return layer
+    }
+    
+    ///
+    /// Apply the forward pass of the Gradient Checking in CPU execution context.
+    ///
+    /// Throw an error if batch size is greater than the first batch size.
+    ///
+    public override func forwardGCCPU() throws
+    {
+        try checkStateCPU(batchSize: batchSize)
+        
+        let (nbSameElems, layersIndex, nbElems) = getMergedGraph()
+        
+        var nbGC = nbSameElems
+        for nbElemsTmp in nbElems
+        {
+            nbGC += nbElemsTmp
+        }
+        
+        for seq in 0..<sequence {
+        for depth in 0..<nbNeurons
+        {
+            neurons.get(seq, depth)!.initGC(batchSize: batchSize, nbGC: nbGC)
+        }}
+        
+        let value = (_layersPrev[0] as! LayerSeq).neurons!
+        let score = (_layersPrev[1] as! LayerSeq).neurons!
+        
+        let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsValue
+        let nbBlocksHead = _nbHeadsScore / _nbHeadsValue
+        
+        for batch in 0..<batchSize {
+        for headScore in 0..<_nbHeadsScore {
+        let headValue = headScore / nbBlocksHead
+        for seqQ in 0..<sequence {
+        for j in 0..<size {
+        for elem in 0..<nbSameElems
+        {
+            let depthScore = j + headScore * size
+            let depthValue = j + headValue * size
+            
+            var sum = 0.0
+            for seqK in 0...seqQ
+            {
+                let valueTmp = value.get(seqK, depthValue)!.gc[batch][elem].out
+                let scoreTmp = score
+                    .get(seqQ, seqK + headScore * sequence)!.gc[batch][elem].out
+                
+                sum += valueTmp * scoreTmp
+            }
+            
+            neurons.get(seqQ, depthScore)!.gc[batch][elem].out = sum
+        }}}}}
+        
+        for batch in 0..<batchSize {
+        for headScore in 0..<_nbHeadsScore {
+        let headValue = headScore / nbBlocksHead
+        for seqQ in 0..<sequence {
+        for j in 0..<size {
+        let depthScore = j + headScore * size
+        let depthValue = j + headValue * size
+        var offset = nbSameElems
+        var nbLastElems = [Int](repeating: nbSameElems,
+                                count: _layersPrev.count)
+        for (index, nbElemsTmp) in zip(layersIndex, nbElems) {
+        for elem in 0..<nbElemsTmp
+        {
+            var sum = 0.0
+            for seqK in 0...seqQ
+            {
+                let valueTmp: Double
+                let scoreTmp: Double
+                
+                if index == 0
+                {
+                    valueTmp = value.get(seqK, depthValue)!
+                        .gc[batch][nbLastElems[index]+elem].out
+                    scoreTmp = score
+                        .get(seqQ, seqK + headScore * sequence)!.v[batch].out
+                }
+                else
+                {
+                    valueTmp = value.get(seqK, depthValue)!.v[batch].out
+                    scoreTmp = score.get(seqQ, seqK + headScore * sequence)!
+                        .gc[batch][nbLastElems[index]+elem].out
+                }
+                
+                sum += valueTmp * scoreTmp
+            }
+            
+            neurons.get(seqQ, depthScore)!.gc[batch][offset+elem].out = sum
+        }
+        
+        offset += nbElemsTmp
+        nbLastElems[index] += nbElemsTmp
+        }}}}}
+    }
+    
+    ///
+    /// Apply the forward pass of the Gradient Checking in GPU execution context.
+    ///
+    /// Throw an error if batch size is greater than the first batch size.
+    ///
+    public override func forwardGCGPU() throws
+    {
+        try checkStateCPU(batchSize: batchSize)
+        
+        let (nbSameElems, layersIndex, nbElems) = getMergedGraph()
+        
+        var nbGC = nbSameElems
+        for nbElemsTmp in nbElems
+        {
+            nbGC += nbElemsTmp
+        }
+        
+        for seq in 0..<sequence {
+        for depth in 0..<nbNeurons
+        {
+            neurons.get(seq, depth)!.initGC(batchSize: batchSize, nbGC: nbGC)
+        }}
+        
+        let value = (_layersPrev[0] as! LayerSeq).neurons!
+        let score = (_layersPrev[1] as! LayerSeq).neurons!
+        
+        let nbNeuronsPrevValue = (_layersPrev[0] as! LayerSeq).nbNeurons
+        let nbNeuronsPrevScore = (_layersPrev[1] as! LayerSeq).nbNeurons
+        
+        let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsValue
+        let nbBlocksHead = _nbHeadsScore / _nbHeadsValue
+        
+        for batch in 0..<batchSize {
+        for headScore in 0..<_nbHeadsScore {
+        let headValue = headScore / nbBlocksHead
+        for seqQ in 0..<sequence {
+        for j in 0..<size {
+        for elem in 0..<nbSameElems
+        {
+            let depthScore = j + headScore * size
+            let depthValue = j + headValue * size
+            
+            var sum = 0.0
+            for seqK in 0...seqQ
+            {
+                let valueTmp = value.get(seqK, depthValue)!.gc[batch][elem].out
+                let scoreTmp = score
+                    .get(seqQ, seqK + headScore * sequence)!.gc[batch][elem].out
+                
+                sum += valueTmp * scoreTmp
+            }
+            
+            neurons.get(seqQ, depthScore)!.gc[batch][elem].out = sum
+        }}}}}
+        
+        let valueBuffer = (_layersPrev[0] as! LayerSeq).outs.download()
+        let scoreBuffer = (_layersPrev[1] as! LayerSeq).outs.download()
+        
+        for batch in 0..<batchSize {
+        for headScore in 0..<_nbHeadsScore {
+        let headValue = headScore / nbBlocksHead
+        for seqQ in 0..<sequence {
+        for j in 0..<size {
+        let depthScore = j + headScore * size
+        let depthValue = j + headValue * size
+        var offset = nbSameElems
+        var nbLastElems = [Int](repeating: nbSameElems,
+                                count: _layersPrev.count)
+        for (index, nbElemsTmp) in zip(layersIndex, nbElems) {
+        for elem in 0..<nbElemsTmp
+        {
+            var sum = 0.0
+            for seqK in 0...seqQ
+            {
+                let valueTmp: Double
+                let scoreTmp: Double
+                
+                if index == 0
+                {
+                    valueTmp = value.get(seqK, depthValue)!
+                        .gc[batch][nbLastElems[index]+elem].out
+                    
+                    let offsetTmp = seqK + headScore * sequence +
+                        nbNeuronsPrevScore * seqQ +
+                        sequence * nbNeuronsPrevScore * batch
+                    
+                    scoreTmp = Double(scoreBuffer[offsetTmp])
+                }
+                else
+                {
+                    let offsetTmp = depthValue + nbNeuronsPrevValue * seqK +
+                        sequence * nbNeuronsPrevValue * batch
+                    
+                    valueTmp = Double(valueBuffer[offsetTmp])
+                    
+                    scoreTmp = score.get(seqQ, seqK + headScore * sequence)!
+                        .gc[batch][nbLastElems[index]+elem].out
+                }
+                
+                sum += valueTmp * scoreTmp
+            }
+            
+            neurons.get(seqQ, depthScore)!.gc[batch][offset+elem].out = sum
+        }
+        
+        offset += nbElemsTmp
+        nbLastElems[index] += nbElemsTmp
+        }}}}}
+    }
+    
+    ///
+    /// Apply the forward pass in the CPU execution context.
+    ///
+    /// Throw an error if batch size is greater than the first batch size.
+    ///
+    public override func forwardCPU() throws
+    {
+        try checkStateCPU(batchSize: batchSize)
+        
+        let value = (_layersPrev[0] as! LayerSeq).neurons!
+        let score = (_layersPrev[1] as! LayerSeq).neurons!
+        
+        let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsValue
+        let nbBlocksHead = _nbHeadsScore / _nbHeadsValue
+        
+        for elem in 0..<batchSize {
+        for headScore in 0..<_nbHeadsScore {
+        let headValue = headScore / nbBlocksHead
+        for seqQ in 0..<sequence {
+        for j in 0..<size
+        {
+            let depthScore = j + headScore * size
+            let depthValue = j + headValue * size
+            
+            var sum = 0.0
+            for seqK in 0...seqQ
+            {
+                let valueTmp = value.get(seqK, depthValue)!.v[elem].out
+                let scoreTmp = score
+                    .get(seqQ, seqK + headScore * sequence)!.v[elem].out
+                
+                sum += valueTmp * scoreTmp
+            }
+            
+            neurons.get(seqQ, depthScore)!.v[elem].out = sum
+        }}}}
+    }
+    
+    ///
+    /// Apply the forward pass in the GPU execution context.
+    ///
+    /// Throw an error if batch size is greater than the first batch size.
+    ///
+    public override func forwardGPU() throws
+    {
+        try checkStateForwardGPU(batchSize: batchSize)
+        
+        let value = _layersPrev[0] as! LayerSeq
+        let score = _layersPrev[1] as! LayerSeq
+        let nbNeuronsPrevValue = value.nbNeurons
+        let nbNeuronsPrevScore = score.nbNeurons
+        
+        let pNbHeadsValue: [UInt32] = [UInt32(_nbHeadsValue)]
+        let pNbHeadsScore: [UInt32] = [UInt32(_nbHeadsScore)]
+        let pNbNeurons: [UInt32] = [UInt32(nbNeurons)]
+        let pNbNeuronsPrevValue: [UInt32] = [UInt32(nbNeuronsPrevValue)]
+        let pNbNeuronsPrevScore: [UInt32] = [UInt32(nbNeuronsPrevScore)]
+        let pNbBatch: [UInt32] = [UInt32(batchSize)]
+        let pSequence: [UInt32] = [UInt32(sequence)]
+        
+        let kernel = (nbNeurons / _nbHeadsScore) % 4 == 0 ?
+            "valueCausalSeq4Forward" : "valueCausalSeqForward"
+        let coeff = (nbNeurons / _nbHeadsScore) % 4 == 0 ? 4 : 1
+        let command = MetalKernel.get.createCommand(
+            kernel, deviceID: deviceID
+        )
+        command.setBuffer(value.outs.metal, atIndex: 0)
+        command.setBuffer(score.outs.metal, atIndex: 1)
+        command.setBytes(pNbHeadsValue, atIndex: 2)
+        command.setBytes(pNbHeadsScore, atIndex: 3)
+        command.setBytes(pNbNeurons, atIndex: 4)
+        command.setBytes(pNbNeuronsPrevValue, atIndex: 5)
+        command.setBytes(pNbNeuronsPrevScore, atIndex: 6)
+        command.setBytes(pNbBatch, atIndex: 7)
+        command.setBytes(pSequence, atIndex: 8)
+        command.setBuffer(outs.metal, atIndex: 9)
+        
+        command.dispatchThreads(
+            width: nbNeurons / coeff,
+            height: batchSize * sequence
+        )
+        command.enqueue()
+    }
+    
+    /// Apply the backward pass in the CPU execution context.
+    public override func backwardCPU()
+    {
+        if !mustComputeBackward
+        {
+            return
+        }
+        
+        let value = (_layersPrev[0] as! LayerSeq).neurons!
+        let score = (_layersPrev[1] as! LayerSeq).neurons!
+        
+        let size = (_layersPrev[0] as! LayerSeq).nbNeurons / _nbHeadsValue
+        let nbBlocksHead = _nbHeadsScore / _nbHeadsValue
+        
+        if _layersPrev[0].computeDelta
+        {
+            for elem in 0..<batchSize {
+            for headValue in 0..<_nbHeadsValue {
+            for seqK in 0..<sequence {
+            for j in 0..<size
+            {
+                let depthValue = j + headValue * size
+                
+                var sum = 0.0
+                for blockHead in 0..<nbBlocksHead
+                {
+                    let headScore = blockHead + nbBlocksHead * headValue
+                    let depthScore = j + headScore * size
+                    
+                    for seqQ in seqK..<sequence
+                    {
+                        let deltaCur = neurons.get(
+                            seqQ, depthScore
+                        )!.v[elem].delta
+                        let scoreTmp = score.get(
+                            seqQ, seqK + headScore * sequence
+                        )!.v[elem].out
+                        
+                        sum += deltaCur * scoreTmp
+                    }
+                }
+                
+                if _layersPrev[0].dirty
+                {
+                    value.get(seqK, depthValue)!.v[elem].delta = sum
+                }
+                else
+                {
+                    value.get(seqK, depthValue)!.v[elem].delta += sum
+                }
+            }}}}
+        }
+        if _layersPrev[1].computeDelta
+        {
+            for elem in 0..<batchSize {
+            for headScore in 0..<_nbHeadsScore {
+            let headValue = headScore / nbBlocksHead
+            for seqQ in 0..<sequence {
+            for seqK in 0...seqQ
+            {
+                var sum = 0.0
+                for j in 0..<size
+                {
+                    let depthScore = j + headScore * size
+                    let depthValue = j + headValue * size
+                    
+                    let deltaCur = neurons.get(seqQ, depthScore)!.v[elem].delta
+                    let valueTmp = value.get(seqK, depthValue)!.v[elem].out
+                    
+                    sum += deltaCur * valueTmp
+                }
+                
+                if _layersPrev[1].dirty
+                {
+                    score.get(seqQ, seqK + headScore * sequence)!
+                        .v[elem].delta = sum
+                }
+                else
+                {
+                    score.get(seqQ, seqK + headScore * sequence)!
+                        .v[elem].delta += sum
+                }
+            }}}}
+        }
+        propagateDirty()
+    }
+    
+    ///
+    /// Apply the backward pass in the GPU execution context.
+    ///
+    /// Throw an error if batch size is greater than the first batch size.
+    ///
+    public override func backwardGPU() throws
+    {
+        if !mustComputeBackward
+        {
+            return
+        }
+        
+        let value = _layersPrev[0] as! LayerSeq
+        let score = _layersPrev[1] as! LayerSeq
+        let nbNeuronsPrevValue = value.nbNeurons
+        let nbNeuronsPrevScore = score.nbNeurons
+        
+        let pNbHeadsValue: [UInt32] = [UInt32(_nbHeadsValue)]
+        let pNbHeadsScore: [UInt32] = [UInt32(_nbHeadsScore)]
+        let pNbNeurons: [UInt32] = [UInt32(nbNeurons)]
+        let pNbNeuronsPrevValue: [UInt32] = [UInt32(nbNeuronsPrevValue)]
+        let pNbNeuronsPrevScore: [UInt32] = [UInt32(nbNeuronsPrevScore)]
+        let pNbBatch: [UInt32] = [UInt32(batchSize)]
+        let pSequence: [UInt32] = [UInt32(sequence)]
+        
+        let metalKernel = MetalKernel.get
+        var command: MetalCommand
+        
+        if value.computeDelta
+        {
+            try value.checkStateBackwardGPU(batchSize: batchSize)
+            
+            let pDirty: [UInt32] = value.dirty ? [1] : [0]
+            
+            let kernel = (nbNeuronsPrevValue / _nbHeadsValue) % 4 == 0 ?
+                "valueCausalValueSeq4Backward" : "valueCausalValueSeqBackward"
+            let coeff = (nbNeuronsPrevValue / _nbHeadsValue) % 4 == 0 ? 4 : 1
+            command = metalKernel.createCommand(
+                kernel, deviceID: deviceID
+            )
+            command.setBuffer(delta.metal, atIndex: 0)
+            command.setBuffer(score.outs.metal, atIndex: 1)
+            command.setBytes(pNbHeadsValue, atIndex: 2)
+            command.setBytes(pNbHeadsScore, atIndex: 3)
+            command.setBytes(pNbNeurons, atIndex: 4)
+            command.setBytes(pNbNeuronsPrevValue, atIndex: 5)
+            command.setBytes(pNbNeuronsPrevScore, atIndex: 6)
+            command.setBytes(pNbBatch, atIndex: 7)
+            command.setBytes(pSequence, atIndex: 8)
+            command.setBytes(pDirty, atIndex: 9)
+            command.setBuffer(value.delta.metal, atIndex: 10)
+            
+            command.dispatchThreads(
+                width: nbNeuronsPrevValue / coeff,
+                height: batchSize * sequence
+            )
+            command.enqueue()
+        }
+        if score.computeDelta
+        {
+            try score.checkStateBackwardGPU(batchSize: batchSize)
+            
+            let pDirty: [UInt32] = score.dirty ? [1] : [0]
+            
+            let kernel = (nbNeurons / _nbHeadsScore) % 4 == 0 ?
+                "valueCausalScoreSeq4Backward" : "valueCausalScoreSeqBackward"
+            command = metalKernel.createCommand(
+                kernel, deviceID: deviceID
+            )
+            command.setBuffer(delta.metal, atIndex: 0)
+            command.setBuffer(value.outs.metal, atIndex: 1)
+            command.setBytes(pNbHeadsValue, atIndex: 2)
+            command.setBytes(pNbHeadsScore, atIndex: 3)
+            command.setBytes(pNbNeurons, atIndex: 4)
+            command.setBytes(pNbNeuronsPrevValue, atIndex: 5)
+            command.setBytes(pNbNeuronsPrevScore, atIndex: 6)
+            command.setBytes(pNbBatch, atIndex: 7)
+            command.setBytes(pSequence, atIndex: 8)
+            command.setBytes(pDirty, atIndex: 9)
+            command.setBuffer(score.delta.metal, atIndex: 10)
+            
+            command.dispatchThreads(
+                width: nbNeuronsPrevScore,
+                height: batchSize * sequence
+            )
+            command.enqueue()
+        }
+        propagateDirty()
+    }
+}
diff --git a/Sources/GrAIdient/Metal/Kernel/NLPFloat.metal b/Sources/GrAIdient/Metal/Kernel/NLPFloat.metal
index 89ad05c7..b3a884c7 100644
--- a/Sources/GrAIdient/Metal/Kernel/NLPFloat.metal
+++ b/Sources/GrAIdient/Metal/Kernel/NLPFloat.metal
@@ -171,6 +171,7 @@ kernel void queryCausalSeqForwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / sequence;
     uint seqK = id[0] % sequence;
@@ -183,8 +184,7 @@ kernel void queryCausalSeqForwardFloat(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     float tmp = 0.0;
     
     for (uint j=0; j<size; j++)
@@ -220,6 +220,7 @@ kernel void queryCausalSeq4ForwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / sequence;
     uint seqK = id[0] % sequence;
@@ -232,8 +233,7 @@ kernel void queryCausalSeq4ForwardFloat(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     float4 tmp = 0.0;
     
     for (uint j=0; j<size/4; j++)
@@ -271,6 +271,7 @@ kernel void queryCausalQuerySeqBackwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / size;
     uint j = id[0] % size;
@@ -283,8 +284,7 @@ kernel void queryCausalQuerySeqBackwardFloat(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     uint depthPrevKey = j + headKey * size;
     uint depthPrevQuery = j + headQuery * size;
     
@@ -328,6 +328,7 @@ kernel void queryCausalQuerySeq4BackwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / (size / 4);
     uint j = id[0] % (size / 4);
@@ -340,8 +341,7 @@ kernel void queryCausalQuerySeq4BackwardFloat(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     uint depthPrevKey = j * 4 + headKey * size;
     uint depthPrevQuery = j * 4 + headQuery * size;
     
@@ -385,6 +385,7 @@ kernel void queryCausalKeySeqBackwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevKey / nbHeadsKey;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headKey = id[0] / size;
     uint j = id[0] % size;
@@ -397,8 +398,6 @@ kernel void queryCausalKeySeqBackwardFloat(
         return ;
     }
     
-    uint nbBlocksHead = nbHeadsQuery == nbHeadsKey ?
-        1 : nbHeadsQuery / nbHeadsKey;
     uint depthPrevKey = j + headKey * size;
     
     float tmp = 0.0;
@@ -448,6 +447,7 @@ kernel void queryCausalKeySeq4BackwardFloat(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevKey / nbHeadsKey;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headKey = id[0] / (size / 4);
     uint j = id[0] % (size / 4);
@@ -460,8 +460,6 @@ kernel void queryCausalKeySeq4BackwardFloat(
         return ;
     }
     
-    uint nbBlocksHead = nbHeadsQuery == nbHeadsKey ?
-        1 : nbHeadsQuery / nbHeadsKey;
     uint depthPrevKey = j * 4 + headKey * size;
     
     float4 tmp = 0.0;
@@ -495,3 +493,334 @@ kernel void queryCausalKeySeq4BackwardFloat(
         key[offsetKey] += tmp;
     }
 }
+
+kernel void valueCausalSeqForwardFloat(
+    const device float * value,
+    const device float * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    device float * outs,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / size;
+    uint j = id[0] % size;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || j >= size ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    uint depthScore = j + headScore * size;
+    uint depthValue = j + headValue * size;
+    
+    float tmp = 0.0;
+    for (uint seqK=0; seqK<=seqQ; seqK++)
+    {
+        uint offsetValue = depthValue +
+            nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+        uint offsetScore = seqK + headScore * sequence +
+            nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+        
+        tmp += value[offsetValue] * score[offsetScore];
+    }
+    
+    uint offset = depthScore + nbNeurons * seqQ + sequence * nbNeurons * elem;
+    outs[offset] = tmp;
+}
+
+kernel void valueCausalSeq4ForwardFloat(
+    const device float4 * value,
+    const device float * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    device float4 * outs,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / (size / 4);
+    uint j = id[0] % (size / 4);
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || j >= size ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    uint depthScore = j * 4 + headScore * size;
+    uint depthValue = j * 4 + headValue * size;
+    
+    float4 tmp = 0.0;
+    for (uint seqK=0; seqK<=seqQ; seqK++)
+    {
+        uint offsetValue = (depthValue +
+            nbNeuronsPrevValue * seqK +
+            sequence * nbNeuronsPrevValue * elem) / 4;
+        uint offsetScore = seqK + headScore * sequence +
+            nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+        
+        tmp += value[offsetValue] * score[offsetScore];
+    }
+    
+    uint offset = (depthScore +
+        nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+    outs[offset] = tmp;
+}
+
+kernel void valueCausalValueSeqBackwardFloat(
+    const device float * delta,
+    const device float * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device float * value,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headValue = id[0] / size;
+    uint j = id[0] % size;
+    uint elem = id[1] / sequence;
+    uint seqK = id[1] % sequence;
+    
+    if (headValue >= nbHeadsValue || j >= size ||
+        elem >= nbBatch || seqK >= sequence)
+    {
+        return ;
+    }
+    
+    uint depthValue = j + headValue * size;
+    
+    float tmp = 0.0;
+    for (uint blockHead=0; blockHead<nbBlocksHead; blockHead++)
+    {
+        uint headScore = blockHead + nbBlocksHead * headValue;
+        uint depthScore = j + headScore * size;
+        
+        for (uint seqQ=seqK; seqQ<sequence; seqQ++)
+        {
+            uint offset = depthScore +
+                nbNeurons * seqQ + sequence * nbNeurons * elem;
+            uint offsetScore = seqK + headScore * sequence +
+                nbNeuronsPrevScore * seqQ +
+                sequence * nbNeuronsPrevScore * elem;
+            
+            tmp += delta[offset] * score[offsetScore];
+        }
+    }
+    
+    uint offsetValue = depthValue +
+        nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+    if (dirty)
+    {
+        value[offsetValue] = tmp;
+    }
+    else
+    {
+        value[offsetValue] += tmp;
+    }
+}
+
+kernel void valueCausalValueSeq4BackwardFloat(
+    const device float4 * delta,
+    const device float * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device float4 * value,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headValue = id[0] / (size / 4);
+    uint j = id[0] % (size / 4);
+    uint elem = id[1] / sequence;
+    uint seqK = id[1] % sequence;
+    
+    if (headValue >= nbHeadsValue || j * 4 >= size ||
+        elem >= nbBatch || seqK >= sequence)
+    {
+        return ;
+    }
+    
+    uint depthValue = j + headValue * size;
+    
+    float4 tmp = 0.0;
+    for (uint blockHead=0; blockHead<nbBlocksHead; blockHead++)
+    {
+        uint headScore = blockHead + nbBlocksHead * headValue;
+        uint depthScore = j * 4 + headScore * size;
+        
+        for (uint seqQ=seqK; seqQ<sequence; seqQ++)
+        {
+            uint offset = (depthScore +
+                nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+            uint offsetScore = seqK + headScore * sequence +
+                nbNeuronsPrevScore * seqQ +
+                sequence * nbNeuronsPrevScore * elem;
+            
+            tmp += delta[offset] * score[offsetScore];
+        }
+    }
+    
+    uint offsetValue = (depthValue +
+        nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem) / 4;
+    if (dirty)
+    {
+        value[offsetValue] = tmp;
+    }
+    else
+    {
+        value[offsetValue] += tmp;
+    }
+}
+
+kernel void valueCausalScoreSeqBackwardFloat(
+    const device float * delta,
+    const device float * value,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device float * score,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / sequence;
+    uint seqK = id[0] % sequence;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || seqK > seqQ ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    float tmp = 0.0;
+    for (uint j=0; j<size; j++)
+    {
+        uint depthScore = j + headScore * size;
+        uint depthValue = j + headValue * size;
+        
+        uint offset = depthScore +
+            nbNeurons * seqQ + sequence * nbNeurons * elem;
+        uint offsetValue = depthValue +
+            nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+        
+        tmp += delta[offset] * value[offsetValue];
+    }
+    
+    uint offsetScore = seqK + headScore * sequence +
+        nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+    
+    if (dirty)
+    {
+        score[offsetScore] = tmp;
+    }
+    else
+    {
+        score[offsetScore] += tmp;
+    }
+}
+
+kernel void valueCausalScoreSeq4BackwardFloat(
+    const device float4 * delta,
+    const device float4 * value,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device float * score,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / sequence;
+    uint seqK = id[0] % sequence;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || seqK > seqQ ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    float4 tmp = 0.0;
+    for (uint j=0; j<size/4; j++)
+    {
+        uint depthScore = j * 4 + headScore * size;
+        uint depthValue = j * 4 + headValue * size;
+        
+        uint offset = (depthScore +
+            nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+        uint offsetValue = (depthValue +
+            nbNeuronsPrevValue * seqK +
+            sequence * nbNeuronsPrevValue * elem) / 4;
+        
+        tmp += delta[offset] * value[offsetValue];
+    }
+    
+    uint offsetScore = seqK + headScore * sequence +
+        nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+    
+    if (dirty)
+    {
+        score[offsetScore] = tmp[0] + tmp[1] + tmp[2] + tmp[3];
+    }
+    else
+    {
+        score[offsetScore] += tmp[0] + tmp[1] + tmp[2] + tmp[3];
+    }
+}
diff --git a/Sources/GrAIdient/Metal/Kernel/NLPHalf.metal b/Sources/GrAIdient/Metal/Kernel/NLPHalf.metal
index fea368fa..a41bc69d 100644
--- a/Sources/GrAIdient/Metal/Kernel/NLPHalf.metal
+++ b/Sources/GrAIdient/Metal/Kernel/NLPHalf.metal
@@ -171,6 +171,7 @@ kernel void queryCausalSeqForwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / sequence;
     uint seqK = id[0] % sequence;
@@ -183,8 +184,7 @@ kernel void queryCausalSeqForwardHalf(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     half tmp = 0.0;
     
     for (uint j=0; j<size; j++)
@@ -220,6 +220,7 @@ kernel void queryCausalSeq4ForwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / sequence;
     uint seqK = id[0] % sequence;
@@ -232,8 +233,7 @@ kernel void queryCausalSeq4ForwardHalf(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
     half4 tmp = 0.0;
     
     for (uint j=0; j<size/4; j++)
@@ -271,6 +271,7 @@ kernel void queryCausalQuerySeqBackwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / size;
     uint j = id[0] % size;
@@ -283,8 +284,8 @@ kernel void queryCausalQuerySeqBackwardHalf(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
+    
     uint depthPrevKey = j + headKey * size;
     uint depthPrevQuery = j + headQuery * size;
     
@@ -328,6 +329,7 @@ kernel void queryCausalQuerySeq4BackwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevQuery / nbHeadsQuery;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headQuery = id[0] / (size / 4);
     uint j = id[0] % (size / 4);
@@ -340,8 +342,8 @@ kernel void queryCausalQuerySeq4BackwardHalf(
         return ;
     }
     
-    uint headKey = nbHeadsQuery == nbHeadsKey ?
-        headQuery : headQuery / nbHeadsKey;
+    uint headKey = headQuery / nbBlocksHead;
+    
     uint depthPrevKey = j * 4 + headKey * size;
     uint depthPrevQuery = j * 4 + headQuery * size;
     
@@ -385,6 +387,7 @@ kernel void queryCausalKeySeqBackwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevKey / nbHeadsKey;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headKey = id[0] / size;
     uint j = id[0] % size;
@@ -397,8 +400,6 @@ kernel void queryCausalKeySeqBackwardHalf(
         return ;
     }
     
-    uint nbBlocksHead = nbHeadsQuery == nbHeadsKey ?
-        1 : nbHeadsQuery / nbHeadsKey;
     uint depthPrevKey = j + headKey * size;
     
     half tmp = 0.0;
@@ -448,6 +449,7 @@ kernel void queryCausalKeySeq4BackwardHalf(
     uint2 id [[ thread_position_in_grid ]])
 {
     uint size = nbNeuronsPrevKey / nbHeadsKey;
+    uint nbBlocksHead = nbHeadsQuery / nbHeadsKey;
     
     uint headKey = id[0] / (size / 4);
     uint j = id[0] % (size / 4);
@@ -460,8 +462,6 @@ kernel void queryCausalKeySeq4BackwardHalf(
         return ;
     }
     
-    uint nbBlocksHead = nbHeadsQuery == nbHeadsKey ?
-        1 : nbHeadsQuery / nbHeadsKey;
     uint depthPrevKey = j * 4 + headKey * size;
     
     half4 tmp = 0.0;
@@ -495,3 +495,334 @@ kernel void queryCausalKeySeq4BackwardHalf(
         key[offsetKey] += tmp;
     }
 }
+
+kernel void valueCausalSeqForwardHalf(
+    const device half * value,
+    const device half * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    device half * outs,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / size;
+    uint j = id[0] % size;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || j >= size ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    uint depthScore = j + headScore * size;
+    uint depthValue = j + headValue * size;
+    
+    half tmp = 0.0;
+    for (uint seqK=0; seqK<=seqQ; seqK++)
+    {
+        uint offsetValue = depthValue +
+            nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+        uint offsetScore = seqK + headScore * sequence +
+            nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+        
+        tmp += value[offsetValue] * score[offsetScore];
+    }
+    
+    uint offset = depthScore + nbNeurons * seqQ + sequence * nbNeurons * elem;
+    outs[offset] = tmp;
+}
+
+kernel void valueCausalSeq4ForwardHalf(
+    const device half4 * value,
+    const device half * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    device half4 * outs,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / (size / 4);
+    uint j = id[0] % (size / 4);
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || j >= size ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    uint depthScore = j * 4 + headScore * size;
+    uint depthValue = j * 4 + headValue * size;
+    
+    half4 tmp = 0.0;
+    for (uint seqK=0; seqK<=seqQ; seqK++)
+    {
+        uint offsetValue = (depthValue +
+            nbNeuronsPrevValue * seqK +
+            sequence * nbNeuronsPrevValue * elem) / 4;
+        uint offsetScore = seqK + headScore * sequence +
+            nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+        
+        tmp += value[offsetValue] * score[offsetScore];
+    }
+    
+    uint offset = (depthScore +
+        nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+    outs[offset] = tmp;
+}
+
+kernel void valueCausalValueSeqBackwardHalf(
+    const device half * delta,
+    const device half * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device half * value,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headValue = id[0] / size;
+    uint j = id[0] % size;
+    uint elem = id[1] / sequence;
+    uint seqK = id[1] % sequence;
+    
+    if (headValue >= nbHeadsValue || j >= size ||
+        elem >= nbBatch || seqK >= sequence)
+    {
+        return ;
+    }
+    
+    uint depthValue = j + headValue * size;
+    
+    half tmp = 0.0;
+    for (uint blockHead=0; blockHead<nbBlocksHead; blockHead++)
+    {
+        uint headScore = blockHead + nbBlocksHead * headValue;
+        uint depthScore = j + headScore * size;
+        
+        for (uint seqQ=seqK; seqQ<sequence; seqQ++)
+        {
+            uint offset = depthScore +
+                nbNeurons * seqQ + sequence * nbNeurons * elem;
+            uint offsetScore = seqK + headScore * sequence +
+                nbNeuronsPrevScore * seqQ +
+                sequence * nbNeuronsPrevScore * elem;
+            
+            tmp += delta[offset] * score[offsetScore];
+        }
+    }
+    
+    uint offsetValue = depthValue +
+        nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+    if (dirty)
+    {
+        value[offsetValue] = tmp;
+    }
+    else
+    {
+        value[offsetValue] += tmp;
+    }
+}
+
+kernel void valueCausalValueSeq4BackwardHalf(
+    const device half4 * delta,
+    const device half * score,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device half4 * value,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headValue = id[0] / (size / 4);
+    uint j = id[0] % (size / 4);
+    uint elem = id[1] / sequence;
+    uint seqK = id[1] % sequence;
+    
+    if (headValue >= nbHeadsValue || j * 4 >= size ||
+        elem >= nbBatch || seqK >= sequence)
+    {
+        return ;
+    }
+    
+    uint depthValue = j + headValue * size;
+    
+    half4 tmp = 0.0;
+    for (uint blockHead=0; blockHead<nbBlocksHead; blockHead++)
+    {
+        uint headScore = blockHead + nbBlocksHead * headValue;
+        uint depthScore = j * 4 + headScore * size;
+        
+        for (uint seqQ=seqK; seqQ<sequence; seqQ++)
+        {
+            uint offset = (depthScore +
+                nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+            uint offsetScore = seqK + headScore * sequence +
+                nbNeuronsPrevScore * seqQ +
+                sequence * nbNeuronsPrevScore * elem;
+            
+            tmp += delta[offset] * score[offsetScore];
+        }
+    }
+    
+    uint offsetValue = (depthValue +
+        nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem) / 4;
+    if (dirty)
+    {
+        value[offsetValue] = tmp;
+    }
+    else
+    {
+        value[offsetValue] += tmp;
+    }
+}
+
+kernel void valueCausalScoreSeqBackwardHalf(
+    const device half * delta,
+    const device half * value,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device half * score,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / sequence;
+    uint seqK = id[0] % sequence;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || seqK > seqQ ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    half tmp = 0.0;
+    for (uint j=0; j<size; j++)
+    {
+        uint depthScore = j + headScore * size;
+        uint depthValue = j + headValue * size;
+        
+        uint offset = depthScore +
+            nbNeurons * seqQ + sequence * nbNeurons * elem;
+        uint offsetValue = depthValue +
+            nbNeuronsPrevValue * seqK + sequence * nbNeuronsPrevValue * elem;
+        
+        tmp += delta[offset] * value[offsetValue];
+    }
+    
+    uint offsetScore = seqK + headScore * sequence +
+        nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+    
+    if (dirty)
+    {
+        score[offsetScore] = tmp;
+    }
+    else
+    {
+        score[offsetScore] += tmp;
+    }
+}
+
+kernel void valueCausalScoreSeq4BackwardHalf(
+    const device half4 * delta,
+    const device half4 * value,
+    constant uint & nbHeadsValue,
+    constant uint & nbHeadsScore,
+    constant uint & nbNeurons,
+    constant uint & nbNeuronsPrevValue,
+    constant uint & nbNeuronsPrevScore,
+    constant uint & nbBatch,
+    constant uint & sequence,
+    constant uint & dirty,
+    device half * score,
+    uint2 id [[ thread_position_in_grid ]])
+{
+    uint size = nbNeuronsPrevValue / nbHeadsValue;
+    uint nbBlocksHead = nbHeadsScore / nbHeadsValue;
+    
+    uint headScore = id[0] / sequence;
+    uint seqK = id[0] % sequence;
+    uint elem = id[1] / sequence;
+    uint seqQ = id[1] % sequence;
+    
+    if (headScore >= nbHeadsScore || seqK > seqQ ||
+        elem >= nbBatch || seqQ >= sequence)
+    {
+        return ;
+    }
+    
+    uint headValue = headScore / nbBlocksHead;
+    
+    half4 tmp = 0.0;
+    for (uint j=0; j<size/4; j++)
+    {
+        uint depthScore = j * 4 + headScore * size;
+        uint depthValue = j * 4 + headValue * size;
+        
+        uint offset = (depthScore +
+            nbNeurons * seqQ + sequence * nbNeurons * elem) / 4;
+        uint offsetValue = (depthValue +
+            nbNeuronsPrevValue * seqK +
+            sequence * nbNeuronsPrevValue * elem) / 4;
+        
+        tmp += delta[offset] * value[offsetValue];
+    }
+    
+    uint offsetScore = seqK + headScore * sequence +
+        nbNeuronsPrevScore * seqQ + sequence * nbNeuronsPrevScore * elem;
+    
+    if (dirty)
+    {
+        score[offsetScore] = tmp[0] + tmp[1] + tmp[2] + tmp[3];
+    }
+    else
+    {
+        score[offsetScore] += tmp[0] + tmp[1] + tmp[2] + tmp[3];
+    }
+}
diff --git a/Sources/GrAIdient/Metal/MetalConfig.swift b/Sources/GrAIdient/Metal/MetalConfig.swift
index 139a0567..c569c1f9 100644
--- a/Sources/GrAIdient/Metal/MetalConfig.swift
+++ b/Sources/GrAIdient/Metal/MetalConfig.swift
@@ -492,6 +492,12 @@ let CONFIG_KERNELS =
         "queryCausalQuerySeq4BackwardFloat",
         "queryCausalKeySeqBackwardFloat",
         "queryCausalKeySeq4BackwardFloat",
+        "valueCausalSeqForwardFloat",
+        "valueCausalSeq4ForwardFloat",
+        "valueCausalValueSeqBackwardFloat",
+        "valueCausalValueSeq4BackwardFloat",
+        "valueCausalScoreSeqBackwardFloat",
+        "valueCausalScoreSeq4BackwardFloat",
     ],
     "NLPHalf": [
         "createRoPESeqMatrixHalf",
@@ -504,6 +510,12 @@ let CONFIG_KERNELS =
         "queryCausalQuerySeq4BackwardHalf",
         "queryCausalKeySeqBackwardHalf",
         "queryCausalKeySeq4BackwardHalf",
+        "valueCausalSeqForwardHalf",
+        "valueCausalSeq4ForwardHalf",
+        "valueCausalValueSeqBackwardHalf",
+        "valueCausalValueSeq4BackwardHalf",
+        "valueCausalScoreSeqBackwardHalf",
+        "valueCausalScoreSeq4BackwardHalf",
     ],
     "OptimizerFloat": [
         "clipGradientsFloat",
diff --git a/Sources/GrAIdient/Utils/Serialization.swift b/Sources/GrAIdient/Utils/Serialization.swift
index ce056948..66870603 100644
--- a/Sources/GrAIdient/Utils/Serialization.swift
+++ b/Sources/GrAIdient/Utils/Serialization.swift
@@ -97,6 +97,7 @@ let LAYER_REGISTRY: [String: Codable.Type] = buildRegistry(
     SelectSeq.self,
     SimilarityBatchError2D.self,
     SimilarityError2D.self,
+    ValueCausalSeq.self,
     ValueSeq.self,
     ValueSelfSeq.self,
     VQ2D.self,
diff --git a/Tests/GrAIExamples/Base/python_lib/nlp/model.py b/Tests/GrAIExamples/Base/python_lib/nlp/model.py
index e7cd4920..567ed4bd 100644
--- a/Tests/GrAIExamples/Base/python_lib/nlp/model.py
+++ b/Tests/GrAIExamples/Base/python_lib/nlp/model.py
@@ -247,11 +247,10 @@ def repeat(a):
             scores.type(torch.float32), dim=-1
         ).type_as(scores)
 
-        """output = torch.matmul(scores, values)
+        output = torch.matmul(scores, values)
         output = output.transpose(1, 2).contiguous().reshape(B, L, -1)
 
-        return self.wo(output), (keys, values)"""
-        return scores.transpose(1, 2).contiguous().reshape(B, L, -1), (keys, values)
+        return self.wo(output), (keys, values)
 
 
 class FeedForward(torch.nn.Module):
diff --git a/Tests/GrAIExamples/NLPExample.swift b/Tests/GrAIExamples/NLPExample.swift
index 26decf00..79f8389d 100644
--- a/Tests/GrAIExamples/NLPExample.swift
+++ b/Tests/GrAIExamples/NLPExample.swift
@@ -16,7 +16,7 @@ final class NLPExample: XCTestCase
     let _modelPath = "TO/UPDATE"
     
     /// Prompt.
-    let _prompt = "Hello"
+    let _prompt = "How do you do?"
     
     /// Initialize test.
     override func setUp()
@@ -86,6 +86,14 @@ final class NLPExample: XCTestCase
             params: params
         )
         
+        let value: LayerSeq = FullyConnectedSeq(
+            layerPrev: layer,
+            nbNeurons: nbHeadsKV * headDim,
+            activation: nil,
+            biases: false,
+            params: params
+        )
+        
         layer = try! QueryCausalSeq(
             query: query, key: key,
             nbHeadsQuery: nbHeadsQuery, nbHeadsKey: nbHeadsKV, 
@@ -97,6 +105,20 @@ final class NLPExample: XCTestCase
             params: params
         )
         
+        layer = try! ValueCausalSeq(
+            value: value, score: layer,
+            nbHeadsValue: nbHeadsKV, nbHeadsScore: nbHeadsQuery,
+            params: params
+        )
+        
+        layer = FullyConnectedSeq(
+            layerPrev: layer,
+            nbNeurons: nbHeadsQuery * headDim,
+            activation: nil,
+            biases: false,
+            params: params
+        )
+        
         /*layer = RMSNormSeq(
             layerPrev: layer,
             activation: nil,
@@ -204,6 +226,10 @@ final class NLPExample: XCTestCase
             else
             {
                 let diffPercent = abs(elemOut - elemRef) / elemRef * 100.0
+                if diffPercent > 1
+                {
+                    print(diffPercent)
+                }
                 XCTAssert(diffPercent < 1)
             }
         }
diff --git a/Tests/GrAITests/Layer1DTests.swift b/Tests/GrAITests/Layer1DTests.swift
index a2dd30d6..6d360574 100644
--- a/Tests/GrAITests/Layer1DTests.swift
+++ b/Tests/GrAITests/Layer1DTests.swift
@@ -593,7 +593,7 @@ class Layer1DFlowPrecisionTests: Layer1DFlowTests
     override func testActivation() throws
     {
         let trainer = _buildTrainer("Activation")
-        run(trainer)
+        run(trainer, diffThreshold: 0.002)
     }
     
     override func testSelectNeurons() throws
diff --git a/Tests/GrAITests/LayerSeqTests.swift b/Tests/GrAITests/LayerSeqTests.swift
index 35d0f408..8598d8e6 100644
--- a/Tests/GrAITests/LayerSeqTests.swift
+++ b/Tests/GrAITests/LayerSeqTests.swift
@@ -881,7 +881,7 @@ class LayerSeqFlowPrecisionTests: LayerSeqFlowTests
     override func testSoftmaxSeq() throws
     {
         let trainer = _buildTrainer("Softmax")
-        run(trainer, diffThreshold: 0.002)
+        run(trainer, diffThreshold: 0.005)
     }
     
     override func testValueSeq() throws
@@ -1339,7 +1339,7 @@ class LayerSeq4FlowPrecisionTests: LayerSeq4FlowTests
     override func testLayerNormSeq() throws
     {
         let trainer = _buildTrainer("LayerNorm")
-        run(trainer, diffThreshold: 0.002)
+        run(trainer, diffThreshold: 0.005)
     }
     
     override func testQuerySeq() throws
diff --git a/Tests/GrAITests/NLPTests.swift b/Tests/GrAITests/NLPTests.swift
index 01372740..41f22b32 100644
--- a/Tests/GrAITests/NLPTests.swift
+++ b/Tests/GrAITests/NLPTests.swift
@@ -126,6 +126,48 @@ class NLPGradTests: EmbeddingSeqMSE1DCase
                 params: params
             )
             
+        case "ValueCausal1":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * 3,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 3, nbHeadsScore: 3,
+                params: params
+            )
+            
+        case "ValueCausal2":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 4 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 2 * 3,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 2, nbHeadsScore: 4,
+                params: params
+            )
+            
         default:
             fatalError("Unreachable.")
         }
@@ -211,6 +253,32 @@ class NLPGradTests: EmbeddingSeqMSE1DCase
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    func testValueCausal1CPU() throws
+    {
+        GrAI.Opti.CPU = true
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    func testValueCausal1GPU() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    func testValueCausal2CPU() throws
+    {
+        GrAI.Opti.CPU = true
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
+    
+    func testValueCausal2GPU() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -322,6 +390,48 @@ class NLPFlowTests: EmbeddingSeqMSE1DCase
                 params: params
             )
             
+        case "ValueCausal1":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * 3,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 3, nbHeadsScore: 3,
+                params: params
+            )
+            
+        case "ValueCausal2":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 4 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 2 * 3,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 2, nbHeadsScore: 4,
+                params: params
+            )
+            
         default:
             fatalError("Unreachable.")
         }
@@ -372,6 +482,18 @@ class NLPFlowTests: EmbeddingSeqMSE1DCase
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -430,6 +552,18 @@ class NLPFlowPrecisionTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer, diffThreshold: 0.002)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -516,6 +650,48 @@ class NLP4FlowTests: EmbeddingSeqMSE1DCase
                 params: params
             )
             
+        case "ValueCausal1":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 3 * 4,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 3, nbHeadsScore: 3,
+                params: params
+            )
+            
+        case "ValueCausal2":
+            let otherLayer: LayerSeq = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 4 * sequence,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = FullyConnectedSeq(
+                layerPrev: layer,
+                nbNeurons: 2 * 4,
+                activation: nil,
+                biases: false,
+                params: params
+            )
+            layer = try! ValueCausalSeq(
+                value: layer, score: otherLayer,
+                nbHeadsValue: 2, nbHeadsScore: 4,
+                params: params
+            )
+            
         default:
             fatalError("Unreachable.")
         }
@@ -541,6 +717,18 @@ class NLP4FlowTests: EmbeddingSeqMSE1DCase
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -574,6 +762,18 @@ class NLP4FlowPrecisionTests: NLP4FlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer, diffThreshold: 0.002)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer, diffThreshold: 0.002)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer, diffThreshold: 0.002)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -640,6 +840,18 @@ class NLPFlowResetTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -706,6 +918,18 @@ class NLPFlowReverseTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -840,6 +1064,18 @@ class NLPInferenceTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -899,6 +1135,18 @@ class NLPLoadTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }
 
 // -----------------------------------------------------------------------------
@@ -1002,4 +1250,16 @@ class NLPTransformTests: NLPFlowTests
         let trainer = _buildTrainer("QueryCausal2")
         run(trainer)
     }
+    
+    override func testValueCausal1() throws
+    {
+        let trainer = _buildTrainer("ValueCausal1")
+        run(trainer)
+    }
+    
+    override func testValueCausal2() throws
+    {
+        let trainer = _buildTrainer("ValueCausal2")
+        run(trainer)
+    }
 }