From 48a3bd3921953971490411daf0c5515d99674600 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Sat, 4 May 2024 15:05:21 +0300 Subject: [PATCH 01/21] init hw7 --- Homework7/Homework7/Homework7.fsproj | 12 ++++++++++++ Homework7/Homework7/Program.fs | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 Homework7/Homework7/Homework7.fsproj create mode 100644 Homework7/Homework7/Program.fs diff --git a/Homework7/Homework7/Homework7.fsproj b/Homework7/Homework7/Homework7.fsproj new file mode 100644 index 0000000..5a3c697 --- /dev/null +++ b/Homework7/Homework7/Homework7.fsproj @@ -0,0 +1,12 @@ + + + + Exe + net8.0 + + + + + + + diff --git a/Homework7/Homework7/Program.fs b/Homework7/Homework7/Program.fs new file mode 100644 index 0000000..d6818ab --- /dev/null +++ b/Homework7/Homework7/Program.fs @@ -0,0 +1,2 @@ +// For more information see https://aka.ms/fsharp-console-apps +printfn "Hello from F#" From 4dd98d24db827a40fd9e224d7ec038c5da51bf08 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 18:39:25 +0300 Subject: [PATCH 02/21] add impl of minicrawler --- Homework7/Homework7/Homework7.fsproj | 2 +- Homework7/Homework7/Library.fs | 45 ++++++++++++++++++++++++++++ Homework7/Homework7/Program.fs | 2 -- 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 Homework7/Homework7/Library.fs delete mode 100644 Homework7/Homework7/Program.fs diff --git a/Homework7/Homework7/Homework7.fsproj b/Homework7/Homework7/Homework7.fsproj index 5a3c697..9c05ed5 100644 --- a/Homework7/Homework7/Homework7.fsproj +++ b/Homework7/Homework7/Homework7.fsproj @@ -6,7 +6,7 @@ - + diff --git a/Homework7/Homework7/Library.fs b/Homework7/Homework7/Library.fs new file mode 100644 index 0000000..8ff4a5c --- /dev/null +++ b/Homework7/Homework7/Library.fs @@ -0,0 +1,45 @@ +module MiniCrawler + +open System.Net.Http +open System.Text.RegularExpressions + +let downloadPageAsync (url: string) = + async { + try + use client = new HttpClient() + let! html = client.GetStringAsync(url) |> Async.AwaitTask + return html + with ex -> + printfn "Failed to download %s: %s" url ex.Message + return "" + } + +let extractLinks (html: string) = + let pattern = @"]*?\s+)?href=([""'])(http[^""']+)\1" + let matches = Regex.Matches(html, pattern) + [ for m in matches -> m.Groups.[2].Value ] + +let downloadPages (url: string) = + async { + let! mainPageHtml = downloadPageAsync url + let linksFromMainPage = extractLinks mainPageHtml + + let downloadTasks = + linksFromMainPage + |> List.map (fun link -> + async { + let! html = downloadPageAsync link + return (link, html.Length) + }) + + return! Async.Parallel downloadTasks + } + +let printSizes (results: (string * int)[]) = + results |> Array.iter (fun (link, size) -> printfn "%s — %d" link size) + +let downloadAndPrintSizes (url: string) = + async { + let! results = downloadPages url + printSizes results + } diff --git a/Homework7/Homework7/Program.fs b/Homework7/Homework7/Program.fs deleted file mode 100644 index d6818ab..0000000 --- a/Homework7/Homework7/Program.fs +++ /dev/null @@ -1,2 +0,0 @@ -// For more information see https://aka.ms/fsharp-console-apps -printfn "Hello from F#" From c2d180761ed5d53ce4b8e26d416165be9f18e53c Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 18:39:34 +0300 Subject: [PATCH 03/21] init tests --- .../Homework7.Tests/Homework7.Tests.fsproj | 24 ++++++++++++++++ Homework7/Homework7.Tests/Program.fs | 4 +++ Homework7/Homework7.Tests/Tests.fs | 8 ++++++ Homework7/Homework7.sln | 28 +++++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 Homework7/Homework7.Tests/Homework7.Tests.fsproj create mode 100644 Homework7/Homework7.Tests/Program.fs create mode 100644 Homework7/Homework7.Tests/Tests.fs create mode 100644 Homework7/Homework7.sln diff --git a/Homework7/Homework7.Tests/Homework7.Tests.fsproj b/Homework7/Homework7.Tests/Homework7.Tests.fsproj new file mode 100644 index 0000000..95a73a0 --- /dev/null +++ b/Homework7/Homework7.Tests/Homework7.Tests.fsproj @@ -0,0 +1,24 @@ + + + net8.0 + false + false + true + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Homework7/Homework7.Tests/Program.fs b/Homework7/Homework7.Tests/Program.fs new file mode 100644 index 0000000..e5bfa1b --- /dev/null +++ b/Homework7/Homework7.Tests/Program.fs @@ -0,0 +1,4 @@ +module Program = + + [] + let main _ = 0 diff --git a/Homework7/Homework7.Tests/Tests.fs b/Homework7/Homework7.Tests/Tests.fs new file mode 100644 index 0000000..1b2d1cf --- /dev/null +++ b/Homework7/Homework7.Tests/Tests.fs @@ -0,0 +1,8 @@ +module Homework7.Tests + +open MiniCrawler +open FsUnit +open NUnit.Framework + +[] +let Test () = Assert.Pass() diff --git a/Homework7/Homework7.sln b/Homework7/Homework7.sln new file mode 100644 index 0000000..6d17a4a --- /dev/null +++ b/Homework7/Homework7.sln @@ -0,0 +1,28 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Homework7", "Homework7\Homework7.fsproj", "{7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Homework7.Tests", "Homework7.Tests\Homework7.Tests.fsproj", "{541551EE-AF15-4DCC-848D-36EDF6A07DD1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}.Release|Any CPU.Build.0 = Release|Any CPU + {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal From fe8ec9526bd8e1ba8617d60990aca3b54c74f651 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 18:43:47 +0300 Subject: [PATCH 04/21] change output type to library --- Homework7/Homework7/Homework7.fsproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Homework7/Homework7/Homework7.fsproj b/Homework7/Homework7/Homework7.fsproj index 9c05ed5..81f0fd5 100644 --- a/Homework7/Homework7/Homework7.fsproj +++ b/Homework7/Homework7/Homework7.fsproj @@ -1,7 +1,7 @@  - Exe + Library net8.0 From cc1b46f3f28e8badf73cd8dce830263b5fe09114 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 19:11:18 +0300 Subject: [PATCH 05/21] refactor --- Homework7/Homework7/Library.fs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Homework7/Homework7/Library.fs b/Homework7/Homework7/Library.fs index 8ff4a5c..d726551 100644 --- a/Homework7/Homework7/Library.fs +++ b/Homework7/Homework7/Library.fs @@ -32,11 +32,12 @@ let downloadPages (url: string) = return (link, html.Length) }) - return! Async.Parallel downloadTasks + let! results = Async.Parallel downloadTasks + return results |> List.ofArray } -let printSizes (results: (string * int)[]) = - results |> Array.iter (fun (link, size) -> printfn "%s — %d" link size) +let printSizes (results: (string * int) list) = + results |> List.iter (fun (link, size) -> printfn "%s — %d" link size) let downloadAndPrintSizes (url: string) = async { From bb1990441a3c794fe30a6b42287474c408bf3d0e Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 19:14:12 +0300 Subject: [PATCH 06/21] change name of project --- Homework7/Homework7.sln | 4 ++-- .../MiniCrawler.Tests.fsproj} | 2 +- Homework7/{Homework7.Tests => MiniCrawler.Tests}/Program.fs | 0 Homework7/{Homework7.Tests => MiniCrawler.Tests}/Tests.fs | 2 +- Homework7/{Homework7 => MiniCrawler}/Library.fs | 0 .../Homework7.fsproj => MiniCrawler/MiniCrawler.fsproj} | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename Homework7/{Homework7.Tests/Homework7.Tests.fsproj => MiniCrawler.Tests/MiniCrawler.Tests.fsproj} (92%) rename Homework7/{Homework7.Tests => MiniCrawler.Tests}/Program.fs (100%) rename Homework7/{Homework7.Tests => MiniCrawler.Tests}/Tests.fs (72%) rename Homework7/{Homework7 => MiniCrawler}/Library.fs (100%) rename Homework7/{Homework7/Homework7.fsproj => MiniCrawler/MiniCrawler.fsproj} (100%) diff --git a/Homework7/Homework7.sln b/Homework7/Homework7.sln index 6d17a4a..a5299bc 100644 --- a/Homework7/Homework7.sln +++ b/Homework7/Homework7.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Homework7", "Homework7\Homework7.fsproj", "{7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}" +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "MiniCrawler", "MiniCrawler\MiniCrawler.fsproj", "{7A2FFF58-1A54-4E4A-A5A7-C0C06E345050}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Homework7.Tests", "Homework7.Tests\Homework7.Tests.fsproj", "{541551EE-AF15-4DCC-848D-36EDF6A07DD1}" +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "MiniCrawler.Tests", "MiniCrawler.Tests\MiniCrawler.Tests.fsproj", "{541551EE-AF15-4DCC-848D-36EDF6A07DD1}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/Homework7/Homework7.Tests/Homework7.Tests.fsproj b/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj similarity index 92% rename from Homework7/Homework7.Tests/Homework7.Tests.fsproj rename to Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj index 95a73a0..13eea0c 100644 --- a/Homework7/Homework7.Tests/Homework7.Tests.fsproj +++ b/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj @@ -19,6 +19,6 @@ - + \ No newline at end of file diff --git a/Homework7/Homework7.Tests/Program.fs b/Homework7/MiniCrawler.Tests/Program.fs similarity index 100% rename from Homework7/Homework7.Tests/Program.fs rename to Homework7/MiniCrawler.Tests/Program.fs diff --git a/Homework7/Homework7.Tests/Tests.fs b/Homework7/MiniCrawler.Tests/Tests.fs similarity index 72% rename from Homework7/Homework7.Tests/Tests.fs rename to Homework7/MiniCrawler.Tests/Tests.fs index 1b2d1cf..5612e94 100644 --- a/Homework7/Homework7.Tests/Tests.fs +++ b/Homework7/MiniCrawler.Tests/Tests.fs @@ -1,4 +1,4 @@ -module Homework7.Tests +module MiniCrawler.Tests open MiniCrawler open FsUnit diff --git a/Homework7/Homework7/Library.fs b/Homework7/MiniCrawler/Library.fs similarity index 100% rename from Homework7/Homework7/Library.fs rename to Homework7/MiniCrawler/Library.fs diff --git a/Homework7/Homework7/Homework7.fsproj b/Homework7/MiniCrawler/MiniCrawler.fsproj similarity index 100% rename from Homework7/Homework7/Homework7.fsproj rename to Homework7/MiniCrawler/MiniCrawler.fsproj From 1e4da4fe154e52433a88041ff5e6119aede0ff5e Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 24 May 2024 19:15:04 +0300 Subject: [PATCH 07/21] init lazy --- Homework7/Homework7.sln | 6 ++++++ Homework7/Lazy/Lazy.fsproj | 12 ++++++++++++ Homework7/Lazy/Program.fs | 2 ++ 3 files changed, 20 insertions(+) create mode 100644 Homework7/Lazy/Lazy.fsproj create mode 100644 Homework7/Lazy/Program.fs diff --git a/Homework7/Homework7.sln b/Homework7/Homework7.sln index a5299bc..20b5ce3 100644 --- a/Homework7/Homework7.sln +++ b/Homework7/Homework7.sln @@ -7,6 +7,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "MiniCrawler", "MiniCrawler\ EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "MiniCrawler.Tests", "MiniCrawler.Tests\MiniCrawler.Tests.fsproj", "{541551EE-AF15-4DCC-848D-36EDF6A07DD1}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Lazy", "Lazy\Lazy.fsproj", "{5310B6BA-3D02-4A8D-A067-A9452C046FC9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -24,5 +26,9 @@ Global {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Debug|Any CPU.Build.0 = Debug|Any CPU {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Release|Any CPU.ActiveCfg = Release|Any CPU {541551EE-AF15-4DCC-848D-36EDF6A07DD1}.Release|Any CPU.Build.0 = Release|Any CPU + {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/Homework7/Lazy/Lazy.fsproj b/Homework7/Lazy/Lazy.fsproj new file mode 100644 index 0000000..5a3c697 --- /dev/null +++ b/Homework7/Lazy/Lazy.fsproj @@ -0,0 +1,12 @@ + + + + Exe + net8.0 + + + + + + + diff --git a/Homework7/Lazy/Program.fs b/Homework7/Lazy/Program.fs new file mode 100644 index 0000000..d6818ab --- /dev/null +++ b/Homework7/Lazy/Program.fs @@ -0,0 +1,2 @@ +// For more information see https://aka.ms/fsharp-console-apps +printfn "Hello from F#" From 385efc2ab4e8c1a591a5a2f9cdb2caa98f3485e3 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Tue, 28 May 2024 15:15:02 +0300 Subject: [PATCH 08/21] add impl of lazy --- Homework7/Lazy/Lazy.fsproj | 4 +-- Homework7/Lazy/Library.fs | 68 ++++++++++++++++++++++++++++++++++++++ Homework7/Lazy/Program.fs | 2 -- 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 Homework7/Lazy/Library.fs delete mode 100644 Homework7/Lazy/Program.fs diff --git a/Homework7/Lazy/Lazy.fsproj b/Homework7/Lazy/Lazy.fsproj index 5a3c697..81f0fd5 100644 --- a/Homework7/Lazy/Lazy.fsproj +++ b/Homework7/Lazy/Lazy.fsproj @@ -1,12 +1,12 @@  - Exe + Library net8.0 - + diff --git a/Homework7/Lazy/Library.fs b/Homework7/Lazy/Library.fs new file mode 100644 index 0000000..a3d3300 --- /dev/null +++ b/Homework7/Lazy/Library.fs @@ -0,0 +1,68 @@ +module Lazy + +open System + +type ILazy<'a> = + abstract member Get: unit -> 'a + +type SingleThreadedLazy<'a>(supplier: unit -> 'a) = + let mutable value: Result<'a, exn> option = None + + interface ILazy<'a> with + member this.Get() = + match value with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> + try + let v = supplier () + value <- Some(Ok v) + v + with ex -> + value <- Some(Error ex) + raise ex + +type ThreadSafeLazy<'a>(supplier: unit -> 'a) = + let mutable value: Result<'a, exn> option = None + let syncObj = obj () + + interface ILazy<'a> with + member this.Get() = + match value with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> + lock syncObj (fun () -> + match value with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> + try + let v = supplier () + value <- Some(Ok v) + v + with ex -> + value <- Some(Error ex) + raise ex) + +type LockFreeLazy<'a>(supplier: unit -> 'a) = + let mutable value: Result<'a, exn> option = None + + interface ILazy<'a> with + member this.Get() = + match value with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> + try + let result = supplier () + + match System.Threading.Interlocked.CompareExchange(&value, Some(Ok result), None) with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> result + with ex -> + System.Threading.Interlocked.CompareExchange(&value, Some(Error ex), None) + |> ignore + + raise ex diff --git a/Homework7/Lazy/Program.fs b/Homework7/Lazy/Program.fs deleted file mode 100644 index d6818ab..0000000 --- a/Homework7/Lazy/Program.fs +++ /dev/null @@ -1,2 +0,0 @@ -// For more information see https://aka.ms/fsharp-console-apps -printfn "Hello from F#" From 8196111d39225667d52b38c8247c4dbab38f9742 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Tue, 28 May 2024 15:16:28 +0300 Subject: [PATCH 09/21] init tests for lazy --- Homework7/Homework7.sln | 6 ++++++ Homework7/Lazy.Tests/Lazy.Tests.fsproj | 22 ++++++++++++++++++++++ Homework7/Lazy.Tests/Program.fs | 4 ++++ Homework7/Lazy.Tests/Tests.fs | 11 +++++++++++ 4 files changed, 43 insertions(+) create mode 100644 Homework7/Lazy.Tests/Lazy.Tests.fsproj create mode 100644 Homework7/Lazy.Tests/Program.fs create mode 100644 Homework7/Lazy.Tests/Tests.fs diff --git a/Homework7/Homework7.sln b/Homework7/Homework7.sln index 20b5ce3..b105972 100644 --- a/Homework7/Homework7.sln +++ b/Homework7/Homework7.sln @@ -9,6 +9,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "MiniCrawler.Tests", "MiniCr EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Lazy", "Lazy\Lazy.fsproj", "{5310B6BA-3D02-4A8D-A067-A9452C046FC9}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Lazy.Tests", "Lazy.Tests\Lazy.Tests.fsproj", "{C21915B1-5326-48A3-B3B1-7C5E98736BCD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,9 @@ Global {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Debug|Any CPU.Build.0 = Debug|Any CPU {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Release|Any CPU.ActiveCfg = Release|Any CPU {5310B6BA-3D02-4A8D-A067-A9452C046FC9}.Release|Any CPU.Build.0 = Release|Any CPU + {C21915B1-5326-48A3-B3B1-7C5E98736BCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C21915B1-5326-48A3-B3B1-7C5E98736BCD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C21915B1-5326-48A3-B3B1-7C5E98736BCD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C21915B1-5326-48A3-B3B1-7C5E98736BCD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/Homework7/Lazy.Tests/Lazy.Tests.fsproj b/Homework7/Lazy.Tests/Lazy.Tests.fsproj new file mode 100644 index 0000000..d61800c --- /dev/null +++ b/Homework7/Lazy.Tests/Lazy.Tests.fsproj @@ -0,0 +1,22 @@ + + + net8.0 + false + false + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Homework7/Lazy.Tests/Program.fs b/Homework7/Lazy.Tests/Program.fs new file mode 100644 index 0000000..e5bfa1b --- /dev/null +++ b/Homework7/Lazy.Tests/Program.fs @@ -0,0 +1,4 @@ +module Program = + + [] + let main _ = 0 diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs new file mode 100644 index 0000000..079a22f --- /dev/null +++ b/Homework7/Lazy.Tests/Tests.fs @@ -0,0 +1,11 @@ +module Lazy.Tests + +open NUnit.Framework + +[] +let Setup () = + () + +[] +let Test1 () = + Assert.Pass() From 487d075e298c0efd6f0e88893afc2fbdad029e79 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Tue, 28 May 2024 15:20:10 +0300 Subject: [PATCH 10/21] delete unnecessary open --- Homework7/Lazy/Library.fs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Homework7/Lazy/Library.fs b/Homework7/Lazy/Library.fs index a3d3300..f87a71e 100644 --- a/Homework7/Lazy/Library.fs +++ b/Homework7/Lazy/Library.fs @@ -1,7 +1,5 @@ module Lazy -open System - type ILazy<'a> = abstract member Get: unit -> 'a From 921baa2097f977b3f571005835116a6a8d92f9f8 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Tue, 28 May 2024 15:22:20 +0300 Subject: [PATCH 11/21] add imports --- Homework7/Lazy.Tests/Lazy.Tests.fsproj | 8 +++++--- Homework7/Lazy.Tests/Tests.fs | 9 +++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Homework7/Lazy.Tests/Lazy.Tests.fsproj b/Homework7/Lazy.Tests/Lazy.Tests.fsproj index d61800c..d7792f5 100644 --- a/Homework7/Lazy.Tests/Lazy.Tests.fsproj +++ b/Homework7/Lazy.Tests/Lazy.Tests.fsproj @@ -10,10 +10,12 @@ + + - - - + + + diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index 079a22f..e424f14 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -1,11 +1,8 @@ module Lazy.Tests +open Lazy open NUnit.Framework - -[] -let Setup () = - () +open FsUnit [] -let Test1 () = - Assert.Pass() +let Test1 () = Assert.Pass() From e4abb33fc437deb5c11fba1bc1f278d3ab1240e5 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Wed, 29 May 2024 19:46:04 +0300 Subject: [PATCH 12/21] add tests for lazy --- Homework7/Lazy.Tests/Tests.fs | 62 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index e424f14..6020c25 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -4,5 +4,63 @@ open Lazy open NUnit.Framework open FsUnit -[] -let Test1 () = Assert.Pass() +let mutable counter = 0 + +let lazyConstructors = + [ (fun f -> SingleThreadedLazy f :> ILazy) + (fun f -> ThreadSafeLazy f :> ILazy) + (fun f -> LockFreeLazy f :> ILazy) ] + |> List.map (fun f -> TestCaseData(f)) + +let multiThreadLazyConstructors = + [ (fun f -> ThreadSafeLazy f :> ILazy) + (fun f -> LockFreeLazy f :> ILazy) ] + |> List.map (fun f -> TestCaseData(f)) + +[] +let ``Value should compute only once`` (lazyConstructor: (unit -> obj) -> ILazy) = + let supplier () = + counter <- counter + 1 + obj () + + counter <- 0 + + let lazyObject = lazyConstructor supplier + let firstCallValue = lazyObject.Get() + let secondCallValue = lazyObject.Get() + + Assert.That(counter, Is.EqualTo(1)) + + +[] +let ``Computed value should be same for several Gets`` (lazyConstructor: (unit -> obj) -> ILazy) = + let supplier () = 42 :> obj + + let lazyObject = lazyConstructor supplier + let firstCallValue = lazyObject.Get() + let secondCallValue = lazyObject.Get() + + Assert.That(firstCallValue, Is.EqualTo(secondCallValue)) + + +[] +let ``Exception in supplier should be thrown`` (lazyConstructor: (unit -> obj) -> ILazy) = + let lazyObject = lazyConstructor (fun () -> raise (System.Exception())) + Assert.Throws(fun () -> lazyObject.Get() |> ignore) |> ignore + + +[] +let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = + let supplier () = 42 :> obj + + let lazyObject = lazyConstructor supplier + let amountOfThreads = 8 + + let tasksArray = + Seq.init amountOfThreads (fun _ -> async { return lazyObject.Get() }) + + tasksArray + |> Async.Parallel + |> Async.RunSynchronously + |> Seq.forall (fun object -> object = 42) + |> should be True From 2f6b3445dd6dcd26e5e50348dbf7a8abd480ac7d Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Wed, 29 May 2024 20:40:38 +0300 Subject: [PATCH 13/21] add minicrawler tests --- Homework7/MiniCrawler.Tests/Tests.fs | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/Homework7/MiniCrawler.Tests/Tests.fs b/Homework7/MiniCrawler.Tests/Tests.fs index 5612e94..c9e9006 100644 --- a/Homework7/MiniCrawler.Tests/Tests.fs +++ b/Homework7/MiniCrawler.Tests/Tests.fs @@ -5,4 +5,36 @@ open FsUnit open NUnit.Framework [] -let Test () = Assert.Pass() +let ``extractLinks should test`` () = + let url = "https://example.com" + + async { + let! htmlPage = downloadPageAsync url + let actualLinks = extractLinks htmlPage + let expectedLinks = [ "https://www.iana.org/domains/example" ] + Assert.AreEqual(expectedLinks, actualLinks) + } + +[] +let ``downloadPages test`` () = + let url = "https://se.math.spbu.ru" + + async { + let! actual = downloadPages url + printfn "%A" actual + + let expected = + [ ("https://spbu.ru/", 112326) + ("https://oops.math.spbu.ru/SE/alumni", 49175) + ("https://ru.wikipedia.org/wiki/%D0%A2%D0%B5%D1%80%D0%B5%D1%85%D0%BE%D0%B2,_%D0%90%D0%BD%D0%B4%D1%80%D0%B5%D0%B9_%D0%9D%D0%B8%D0%BA%D0%BE%D0%BB%D0%B0%D0%B5%D0%B2%D0%B8%D1%87", + 102683) + ("https://ru.wikipedia.org/wiki/%D0%A2%D0%B5%D1%80%D0%B5%D1%85%D0%BE%D0%B2,_%D0%90%D0%BD%D0%B4%D1%80%D0%B5%D0%B9_%D0%9D%D0%B8%D0%BA%D0%BE%D0%BB%D0%B0%D0%B5%D0%B2%D0%B8%D1%87", + 102683) + ("https://www.acm.org/binaries/content/assets/education/curricula-recommendations/cc2005-march06final.pdf", + 758040) + ("https://t.me/sysprog_admission", 12477) + ("https://oops.math.spbu.ru/SE", 23408) + ("https://oops.math.spbu.ru/SE/alumni", 49175) ] + + Assert.AreEqual(expected, actual) + } From b69666a704b2cebaf35a1ea89ef1077c43be1481 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Wed, 29 May 2024 20:50:02 +0300 Subject: [PATCH 14/21] delete bad test --- Homework7/MiniCrawler.Tests/Tests.fs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/Homework7/MiniCrawler.Tests/Tests.fs b/Homework7/MiniCrawler.Tests/Tests.fs index c9e9006..2ebe9cc 100644 --- a/Homework7/MiniCrawler.Tests/Tests.fs +++ b/Homework7/MiniCrawler.Tests/Tests.fs @@ -14,27 +14,3 @@ let ``extractLinks should test`` () = let expectedLinks = [ "https://www.iana.org/domains/example" ] Assert.AreEqual(expectedLinks, actualLinks) } - -[] -let ``downloadPages test`` () = - let url = "https://se.math.spbu.ru" - - async { - let! actual = downloadPages url - printfn "%A" actual - - let expected = - [ ("https://spbu.ru/", 112326) - ("https://oops.math.spbu.ru/SE/alumni", 49175) - ("https://ru.wikipedia.org/wiki/%D0%A2%D0%B5%D1%80%D0%B5%D1%85%D0%BE%D0%B2,_%D0%90%D0%BD%D0%B4%D1%80%D0%B5%D0%B9_%D0%9D%D0%B8%D0%BA%D0%BE%D0%BB%D0%B0%D0%B5%D0%B2%D0%B8%D1%87", - 102683) - ("https://ru.wikipedia.org/wiki/%D0%A2%D0%B5%D1%80%D0%B5%D1%85%D0%BE%D0%B2,_%D0%90%D0%BD%D0%B4%D1%80%D0%B5%D0%B9_%D0%9D%D0%B8%D0%BA%D0%BE%D0%BB%D0%B0%D0%B5%D0%B2%D0%B8%D1%87", - 102683) - ("https://www.acm.org/binaries/content/assets/education/curricula-recommendations/cc2005-march06final.pdf", - 758040) - ("https://t.me/sysprog_admission", 12477) - ("https://oops.math.spbu.ru/SE", 23408) - ("https://oops.math.spbu.ru/SE/alumni", 49175) ] - - Assert.AreEqual(expected, actual) - } From 9b32b0c42c62fc9a356ba41e4535b29fdcd4a376 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 31 May 2024 20:03:59 +0300 Subject: [PATCH 15/21] fix test name --- Homework7/MiniCrawler.Tests/Tests.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Homework7/MiniCrawler.Tests/Tests.fs b/Homework7/MiniCrawler.Tests/Tests.fs index 2ebe9cc..b8a1e6b 100644 --- a/Homework7/MiniCrawler.Tests/Tests.fs +++ b/Homework7/MiniCrawler.Tests/Tests.fs @@ -5,7 +5,7 @@ open FsUnit open NUnit.Framework [] -let ``extractLinks should test`` () = +let ``extractLinks test`` () = let url = "https://example.com" async { From 5d7258fc4288d9b7d84f77645feb81c8380c6636 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 31 May 2024 22:44:30 +0300 Subject: [PATCH 16/21] change ints to obj --- Homework7/Lazy.Tests/Tests.fs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index 6020c25..f70602c 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -34,7 +34,7 @@ let ``Value should compute only once`` (lazyConstructor: (unit -> obj) -> ILazy< [] let ``Computed value should be same for several Gets`` (lazyConstructor: (unit -> obj) -> ILazy) = - let supplier () = 42 :> obj + let supplier () = obj () let lazyObject = lazyConstructor supplier let firstCallValue = lazyObject.Get() @@ -51,7 +51,8 @@ let ``Exception in supplier should be thrown`` (lazyConstructor: (unit -> obj) - [] let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = - let supplier () = 42 :> obj + let object = obj () + let supplier () = object let lazyObject = lazyConstructor supplier let amountOfThreads = 8 @@ -62,5 +63,5 @@ let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = tasksArray |> Async.Parallel |> Async.RunSynchronously - |> Seq.forall (fun object -> object = 42) + |> Seq.forall (fun x -> x = object) |> should be True From a2eaec4e31d9cb26cc8c52add2c511f08b68aa34 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Fri, 31 May 2024 23:04:32 +0300 Subject: [PATCH 17/21] fix tests after review --- Homework7/Lazy.Tests/Tests.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index f70602c..bcbbaca 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -51,8 +51,7 @@ let ``Exception in supplier should be thrown`` (lazyConstructor: (unit -> obj) - [] let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = - let object = obj () - let supplier () = object + let supplier () = obj () let lazyObject = lazyConstructor supplier let amountOfThreads = 8 @@ -63,5 +62,6 @@ let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = tasksArray |> Async.Parallel |> Async.RunSynchronously - |> Seq.forall (fun x -> x = object) - |> should be True + |> Seq.distinct + |> Seq.length + |> should equal 1 From 1a956da29880bac68d5d9ebdc2ee1eec6a800ec7 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Sat, 1 Jun 2024 00:53:14 +0300 Subject: [PATCH 18/21] fix lockfreelazy --- Homework7/Lazy/Library.fs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Homework7/Lazy/Library.fs b/Homework7/Lazy/Library.fs index f87a71e..106a6a1 100644 --- a/Homework7/Lazy/Library.fs +++ b/Homework7/Lazy/Library.fs @@ -60,7 +60,9 @@ type LockFreeLazy<'a>(supplier: unit -> 'a) = | Some(Error ex) -> raise ex | None -> result with ex -> - System.Threading.Interlocked.CompareExchange(&value, Some(Error ex), None) - |> ignore - - raise ex + match System.Threading.Interlocked.CompareExchange(&value, Some(Error ex), None) with + | Some(Ok v) -> v + | Some(Error ex) -> raise ex + | None -> + value <- Some(Error ex) + raise ex From 83eb9231904a07f47ea38d8caf0b4215dfcc85e0 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Sat, 1 Jun 2024 00:56:56 +0300 Subject: [PATCH 19/21] add test for computing only once in multithread --- Homework7/Lazy.Tests/Tests.fs | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index bcbbaca..e60d264 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -12,10 +12,6 @@ let lazyConstructors = (fun f -> LockFreeLazy f :> ILazy) ] |> List.map (fun f -> TestCaseData(f)) -let multiThreadLazyConstructors = - [ (fun f -> ThreadSafeLazy f :> ILazy) - (fun f -> LockFreeLazy f :> ILazy) ] - |> List.map (fun f -> TestCaseData(f)) [] let ``Value should compute only once`` (lazyConstructor: (unit -> obj) -> ILazy) = @@ -48,20 +44,22 @@ let ``Exception in supplier should be thrown`` (lazyConstructor: (unit -> obj) - let lazyObject = lazyConstructor (fun () -> raise (System.Exception())) Assert.Throws(fun () -> lazyObject.Get() |> ignore) |> ignore +[] +let ``Multithread lazy test`` () = + let counter = ref 0 -[] -let ``Multithread lazies test`` (lazyConstructor: (unit -> obj) -> ILazy) = - let supplier () = obj () - - let lazyObject = lazyConstructor supplier - let amountOfThreads = 8 + let supplier () = + System.Threading.Interlocked.Increment(counter) |> ignore + obj () - let tasksArray = - Seq.init amountOfThreads (fun _ -> async { return lazyObject.Get() }) + let lazyObject = ThreadSafeLazy supplier :> obj ILazy - tasksArray + Seq.initInfinite (fun _ -> async { return lazyObject.Get() }) + |> Seq.take 8 |> Async.Parallel |> Async.RunSynchronously |> Seq.distinct |> Seq.length |> should equal 1 + + counter.Value |> should equal 1 From a9bf8f71a684ab8b5f48ba3f9631dfd70cfd5da1 Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Sat, 1 Jun 2024 01:03:22 +0300 Subject: [PATCH 20/21] return test for lockfreelazy --- Homework7/Lazy.Tests/Tests.fs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Homework7/Lazy.Tests/Tests.fs b/Homework7/Lazy.Tests/Tests.fs index e60d264..6f3563f 100644 --- a/Homework7/Lazy.Tests/Tests.fs +++ b/Homework7/Lazy.Tests/Tests.fs @@ -12,6 +12,10 @@ let lazyConstructors = (fun f -> LockFreeLazy f :> ILazy) ] |> List.map (fun f -> TestCaseData(f)) +let multiThreadLazyConstructors = + [ (fun f -> ThreadSafeLazy f :> ILazy) + (fun f -> LockFreeLazy f :> ILazy) ] + |> List.map (fun f -> TestCaseData(f)) [] let ``Value should compute only once`` (lazyConstructor: (unit -> obj) -> ILazy) = @@ -44,6 +48,22 @@ let ``Exception in supplier should be thrown`` (lazyConstructor: (unit -> obj) - let lazyObject = lazyConstructor (fun () -> raise (System.Exception())) Assert.Throws(fun () -> lazyObject.Get() |> ignore) |> ignore + +[] +let ``Multithread lazies should return same value`` (lazyConstructor: (unit -> obj) -> ILazy) = + let supplier () = obj () + + let lazyObject = ThreadSafeLazy supplier :> obj ILazy + + Seq.initInfinite (fun _ -> async { return lazyObject.Get() }) + |> Seq.take 8 + |> Async.Parallel + |> Async.RunSynchronously + |> Seq.distinct + |> Seq.length + |> should equal 1 + + [] let ``Multithread lazy test`` () = let counter = ref 0 From 5ad0b6cb1d7de109d35c91e52071600ff9f72acc Mon Sep 17 00:00:00 2001 From: kazbekovruslan Date: Sat, 1 Jun 2024 12:35:57 +0300 Subject: [PATCH 21/21] add tests with moq --- .../MiniCrawler.Tests.fsproj | 1 + Homework7/MiniCrawler.Tests/Tests.fs | 60 ++++++++++++++++++- Homework7/MiniCrawler/Library.fs | 22 ++++--- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj b/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj index 13eea0c..24f94d3 100644 --- a/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj +++ b/Homework7/MiniCrawler.Tests/MiniCrawler.Tests.fsproj @@ -17,6 +17,7 @@ + diff --git a/Homework7/MiniCrawler.Tests/Tests.fs b/Homework7/MiniCrawler.Tests/Tests.fs index b8a1e6b..a24c626 100644 --- a/Homework7/MiniCrawler.Tests/Tests.fs +++ b/Homework7/MiniCrawler.Tests/Tests.fs @@ -3,14 +3,72 @@ module MiniCrawler.Tests open MiniCrawler open FsUnit open NUnit.Framework +open Moq +open System.Net.Http +open System.Net [] let ``extractLinks test`` () = let url = "https://example.com" async { - let! htmlPage = downloadPageAsync url + use client = new HttpClient() + let! htmlPage = downloadPageAsync (url, client) let actualLinks = extractLinks htmlPage let expectedLinks = [ "https://www.iana.org/domains/example" ] Assert.AreEqual(expectedLinks, actualLinks) } + +[] +let ``downloadPages test`` () = + let client = Mock() + let url = "https://example.com/" + + let response = new HttpResponseMessage(HttpStatusCode.OK) + + client + .Setup(fun client -> + client.SendAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(response) + .Callback(fun (req: HttpRequestMessage) (ct: System.Threading.CancellationToken) -> + if (req).RequestUri.AbsoluteUri = url then + response.Content <- new StringContent("Example content") + else + response.Content <- new StringContent($"Example")) + |> ignore + + let expected = [ (url, 15) ] + + let actual = + Async.RunSynchronously(downloadPages ("https://fakeurl.com", client.Object)) + + Assert.AreEqual(expected, actual) + +[] +let ``downloadPages with 1 correct and 1 incorrect links`` () : unit = + let client = Mock() + let url = "https://example.com/" + let urlNotFould = "notFound.com" + + let response = new HttpResponseMessage(HttpStatusCode.OK) + + client + .Setup(fun client -> + client.SendAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(response) + .Callback(fun (req: HttpRequestMessage) (ct: System.Threading.CancellationToken) -> + if (req).RequestUri.AbsoluteUri = url then + response.Content <- new StringContent("Example content") + else + response.Content <- + new StringContent( + $"Example Example" + )) + |> ignore + + let expected = [ (url, 15) ] + + let actual = + Async.RunSynchronously(downloadPages ("https://fakeurl.com", client.Object)) + + Assert.AreEqual(expected, actual) diff --git a/Homework7/MiniCrawler/Library.fs b/Homework7/MiniCrawler/Library.fs index d726551..653809d 100644 --- a/Homework7/MiniCrawler/Library.fs +++ b/Homework7/MiniCrawler/Library.fs @@ -3,12 +3,17 @@ open System.Net.Http open System.Text.RegularExpressions -let downloadPageAsync (url: string) = +let downloadPageAsync (url: string, client: HttpClient) = async { try - use client = new HttpClient() - let! html = client.GetStringAsync(url) |> Async.AwaitTask - return html + let httpRequest = new HttpRequestMessage(HttpMethod.Get, url) + + let! response = + client.SendAsync(httpRequest, System.Threading.CancellationToken.None) + |> Async.AwaitTask + + let! content = response.Content.ReadAsStringAsync() |> Async.AwaitTask + return content with ex -> printfn "Failed to download %s: %s" url ex.Message return "" @@ -19,16 +24,16 @@ let extractLinks (html: string) = let matches = Regex.Matches(html, pattern) [ for m in matches -> m.Groups.[2].Value ] -let downloadPages (url: string) = +let downloadPages (url: string, client: HttpClient) = async { - let! mainPageHtml = downloadPageAsync url + let! mainPageHtml = downloadPageAsync (url, client) let linksFromMainPage = extractLinks mainPageHtml let downloadTasks = linksFromMainPage |> List.map (fun link -> async { - let! html = downloadPageAsync link + let! html = downloadPageAsync (link, client) return (link, html.Length) }) @@ -41,6 +46,7 @@ let printSizes (results: (string * int) list) = let downloadAndPrintSizes (url: string) = async { - let! results = downloadPages url + use client = new HttpClient() + let! results = downloadPages (url, client) printSizes results }