From d39fca821ca2b7ac69dd7e7a3b2af86f2c8295a8 Mon Sep 17 00:00:00 2001 From: Zhenyong Zhu Date: Wed, 25 Jul 2018 16:09:43 -0400 Subject: [PATCH 1/5] Fix concurrency errors with concurrentdictionary --- src/Deedle/FrameUtils.fs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Deedle/FrameUtils.fs b/src/Deedle/FrameUtils.fs index a3662750..7c51d417 100644 --- a/src/Deedle/FrameUtils.fs +++ b/src/Deedle/FrameUtils.fs @@ -14,6 +14,7 @@ module internal Reflection = open Microsoft.FSharp.Quotations open System.Collections open System.Collections.Generic + open System.Collections.Concurrent let indexBuilder = IndexBuilder.Instance let vectorBuilder = VectorBuilder.Instance @@ -67,7 +68,7 @@ module internal Reflection = /// Helper function used when building frames from data tables let createTypedVector : _ -> seq> -> _ = - let cache = Dictionary<_, _>() + let cache = ConcurrentDictionary<_, _>() (fun typ -> match cache.TryGetValue(typ) with | true, res -> res @@ -75,7 +76,9 @@ module internal Reflection = let par = Expression.Parameter(typeof>>) let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) let f = Expression.Lambda>, IVector>>(body, par).Compile() - cache.Add(typ, f.Invoke) + match cache.TryAdd(typ, f.Invoke) with + | true -> () + | false -> () f.Invoke ) let getExpandableProperties (ty:Type) = From 8c18c5b452163f411832fff633218bba1437a7b6 Mon Sep 17 00:00:00 2001 From: Zhenyong Zhu Date: Wed, 25 Jul 2018 16:52:07 -0400 Subject: [PATCH 2/5] use ConcurrentDictionary.GetOrAdd --- src/Deedle/FrameUtils.fs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/Deedle/FrameUtils.fs b/src/Deedle/FrameUtils.fs index 7c51d417..ab866d8b 100644 --- a/src/Deedle/FrameUtils.fs +++ b/src/Deedle/FrameUtils.fs @@ -69,17 +69,11 @@ module internal Reflection = /// Helper function used when building frames from data tables let createTypedVector : _ -> seq> -> _ = let cache = ConcurrentDictionary<_, _>() - (fun typ -> - match cache.TryGetValue(typ) with - | true, res -> res - | false, _ -> - let par = Expression.Parameter(typeof>>) - let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) - let f = Expression.Lambda>, IVector>>(body, par).Compile() - match cache.TryAdd(typ, f.Invoke) with - | true -> () - | false -> () - f.Invoke ) + fun typ -> + let par = Expression.Parameter(typeof>>) + let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) + let f = Expression.Lambda>, IVector>>(body, par).Compile() + cache.GetOrAdd(typ, f.Invoke) let getExpandableProperties (ty:Type) = ty.GetProperties(BindingFlags.Instance ||| BindingFlags.Public) From 73e8cd75a25be4df3d1debc9e26f3725769eac75 Mon Sep 17 00:00:00 2001 From: Zhenyong Zhu Date: Wed, 25 Jul 2018 20:29:26 -0400 Subject: [PATCH 3/5] add back trygetvalue. use concurrent dict in getCachedCompileProjection --- src/Deedle/FrameUtils.fs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Deedle/FrameUtils.fs b/src/Deedle/FrameUtils.fs index ab866d8b..d159f68e 100644 --- a/src/Deedle/FrameUtils.fs +++ b/src/Deedle/FrameUtils.fs @@ -70,10 +70,13 @@ module internal Reflection = let createTypedVector : _ -> seq> -> _ = let cache = ConcurrentDictionary<_, _>() fun typ -> - let par = Expression.Parameter(typeof>>) - let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) - let f = Expression.Lambda>, IVector>>(body, par).Compile() - cache.GetOrAdd(typ, f.Invoke) + match cache.TryGetValue(typ) with + | true, res -> res + | false, _ -> + let par = Expression.Parameter(typeof>>) + let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) + let f = Expression.Lambda>, IVector>>(body, par).Compile() + cache.GetOrAdd(typ, f.Invoke) let getExpandableProperties (ty:Type) = ty.GetProperties(BindingFlags.Instance ||| BindingFlags.Public) @@ -125,15 +128,15 @@ module internal Reflection = /// Compile all projections from the type, so that we can run them fast /// and cache the results with Type as the key, so that we don't have to recompile let getCachedCompileProjection = - let cache = Dictionary<_, _>() + let cache = ConcurrentDictionary<_, _>() (fun typ -> match cache.TryGetValue(typ) with | true, res -> res | _ -> let res = [| for name, fldTy, proj in getMemberProjections typ -> name, fldTy, proj.Compile() |] - cache.Add(typ, res) - res ) + cache.GetOrAdd(typ, res) + ) /// Given a single vector, expand its values into multiple vectors. This may be: /// - `IDictionary` is expanded based on keys/values From d27a4d32496389d1b5f851838df47a574e0d407b Mon Sep 17 00:00:00 2001 From: Zhenyong Zhu Date: Thu, 26 Jul 2018 13:04:29 -0400 Subject: [PATCH 4/5] fix concurrent dict with valuefactory --- src/Deedle/FrameUtils.fs | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/Deedle/FrameUtils.fs b/src/Deedle/FrameUtils.fs index d159f68e..8a427f12 100644 --- a/src/Deedle/FrameUtils.fs +++ b/src/Deedle/FrameUtils.fs @@ -68,15 +68,13 @@ module internal Reflection = /// Helper function used when building frames from data tables let createTypedVector : _ -> seq> -> _ = - let cache = ConcurrentDictionary<_, _>() + let cache = ConcurrentDictionary<_, seq> -> _>() + let valueFactory typ = + let par = Expression.Parameter(typeof>>) + let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) + Expression.Lambda>, _>>(body, par).Compile().Invoke fun typ -> - match cache.TryGetValue(typ) with - | true, res -> res - | false, _ -> - let par = Expression.Parameter(typeof>>) - let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) - let f = Expression.Lambda>, IVector>>(body, par).Compile() - cache.GetOrAdd(typ, f.Invoke) + cache.GetOrAdd(typ, Func<_,_> valueFactory) let getExpandableProperties (ty:Type) = ty.GetProperties(BindingFlags.Instance ||| BindingFlags.Public) @@ -128,15 +126,12 @@ module internal Reflection = /// Compile all projections from the type, so that we can run them fast /// and cache the results with Type as the key, so that we don't have to recompile let getCachedCompileProjection = - let cache = ConcurrentDictionary<_, _>() - (fun typ -> - match cache.TryGetValue(typ) with - | true, res -> res - | _ -> - let res = [| for name, fldTy, proj in getMemberProjections typ -> - name, fldTy, proj.Compile() |] - cache.GetOrAdd(typ, res) - ) + let cache = ConcurrentDictionary<_, (string*Type*Delegate)[]>() + let valueFactory typ = + [| for name, fldTy, proj in getMemberProjections typ -> + name, fldTy, proj.Compile() |] + fun typ -> + cache.GetOrAdd(typ, valueFactory) /// Given a single vector, expand its values into multiple vectors. This may be: /// - `IDictionary` is expanded based on keys/values From 462ab8f1a8899b84a72104170e570c9b411a05b9 Mon Sep 17 00:00:00 2001 From: Zhenyong Zhu Date: Fri, 27 Jul 2018 10:41:07 -0400 Subject: [PATCH 5/5] change Func<_,_> so that delegate is executed only once --- src/Deedle/FrameUtils.fs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Deedle/FrameUtils.fs b/src/Deedle/FrameUtils.fs index 8a427f12..71157330 100644 --- a/src/Deedle/FrameUtils.fs +++ b/src/Deedle/FrameUtils.fs @@ -69,12 +69,13 @@ module internal Reflection = /// Helper function used when building frames from data tables let createTypedVector : _ -> seq> -> _ = let cache = ConcurrentDictionary<_, seq> -> _>() - let valueFactory typ = - let par = Expression.Parameter(typeof>>) - let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) - Expression.Lambda>, _>>(body, par).Compile().Invoke + let valueFactory = + Func<_,_> (fun typ -> + let par = Expression.Parameter(typeof>>) + let body = Expression.Call(createTypedVectorMi.MakeGenericMethod([| typ |]), par) + Expression.Lambda>, _>>(body, par).Compile().Invoke) fun typ -> - cache.GetOrAdd(typ, Func<_,_> valueFactory) + cache.GetOrAdd(typ, valueFactory) let getExpandableProperties (ty:Type) = ty.GetProperties(BindingFlags.Instance ||| BindingFlags.Public) @@ -127,9 +128,10 @@ module internal Reflection = /// and cache the results with Type as the key, so that we don't have to recompile let getCachedCompileProjection = let cache = ConcurrentDictionary<_, (string*Type*Delegate)[]>() - let valueFactory typ = - [| for name, fldTy, proj in getMemberProjections typ -> - name, fldTy, proj.Compile() |] + let valueFactory = + Func<_,_> (fun typ -> + [| for name, fldTy, proj in getMemberProjections typ -> + name, fldTy, proj.Compile() |] ) fun typ -> cache.GetOrAdd(typ, valueFactory)