From 25fefd97a041c8222b3d343ddab4441f7d751a38 Mon Sep 17 00:00:00 2001 From: Taylor Southwick Date: Thu, 17 Sep 2020 17:09:45 -0700 Subject: [PATCH] Abstract file access pattern for DSS provider This is an intial change to abstract file access using IPlateTilePyramid. This interface provides a single method that will take a plate file name along with the desired level and coordinates and will retrieve the associated image. This change includes two implementations of this: - ConfigurationManagerFilePlateTilePyramid allows for a request to a given plate file and returns a stream for the level and coordinates specified. - AzurePlateTilePyramid retrieves the file from Azure blob storage This new access method has been incorporated into the DSS.aspx page and will surface the content via Azure or local storage via a configuration flag (requires a restart of the service). --- WWTMVC5/App_Start/UnityConfig.cs | 18 +++++ WWTMVC5/WWTMVC5.csproj | 10 +-- WWTMVC5/Web.config | 4 + WWTWebSiteOnly.sln | 6 ++ WWTWebservices.Azure/AzurePlateTilePyramid.cs | 35 +++++++++ .../WWTWebservices.Azure.csproj | 19 +++++ ...onfigurationManagerFilePlateTilePyramid.cs | 69 ++++++++++++++++ WWTWebservices/IPlateTilePyramid.cs | 9 +++ WWTWebservices/PlateTilePyramid.cs | 3 +- WWTWebservices/WWTWebservices.csproj | 2 +- src/WWT.Providers/Providers/DSSProvider.cs | 78 ++++++++----------- src/WWT.Providers/QueryRequestProvider.cs | 18 +++++ 12 files changed, 217 insertions(+), 54 deletions(-) create mode 100644 WWTWebservices.Azure/AzurePlateTilePyramid.cs create mode 100644 WWTWebservices.Azure/WWTWebservices.Azure.csproj create mode 100644 WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs create mode 100644 WWTWebservices/IPlateTilePyramid.cs create mode 100644 src/WWT.Providers/QueryRequestProvider.cs diff --git a/WWTMVC5/App_Start/UnityConfig.cs b/WWTMVC5/App_Start/UnityConfig.cs index 7651318f..ef24290e 100644 --- a/WWTMVC5/App_Start/UnityConfig.cs +++ b/WWTMVC5/App_Start/UnityConfig.cs @@ -1,7 +1,12 @@ using System; +using System.Configuration; using System.Linq; +using Azure.Core; +using Azure.Identity; using Microsoft.Practices.Unity; using WWT.Providers; +using WWTWebservices; +using WWTWebservices.Azure; namespace WWTMVC5 { @@ -34,6 +39,7 @@ public static IUnityContainer GetConfiguredContainer() public static void RegisterTypes(IUnityContainer container) { RegisterRequestProviders(container); + RegisterPlateFileProvider(container); // NOTE: To load from web.config uncomment the line below. Make sure to add a Microsoft.Practices.Unity.Configuration to the using statements. // container.LoadConfiguration(); @@ -52,5 +58,17 @@ private static void RegisterRequestProviders(IUnityContainer container) container.RegisterType(type); } } + + private static void RegisterPlateFileProvider(IUnityContainer container) + { + if (ConfigReader.GetSetting("UseAzurePlateFiles")) + { + container.RegisterInstance(new AzurePlateTilePyramid(ConfigurationManager.AppSettings["AzurePlateFileContainer"], new DefaultAzureCredential())); + } + else + { + container.RegisterType(new ContainerControlledLifetimeManager()); + } + } } } diff --git a/WWTMVC5/WWTMVC5.csproj b/WWTMVC5/WWTMVC5.csproj index 393270fa..f275ebb2 100644 --- a/WWTMVC5/WWTMVC5.csproj +++ b/WWTMVC5/WWTMVC5.csproj @@ -1377,6 +1377,10 @@ {ee4a3106-572b-4ef4-9ab5-a22643309a57} WWT.Providers + + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9} + WWTWebservices.Azure + {1cf4d986-51ad-43f8-a84a-38b6ecba2172} WWTWebservices @@ -1443,9 +1447,6 @@ 6.1.7600.16394 - - 3.2.3 - 10.0.3 @@ -1461,9 +1462,6 @@ 1.6.0 - - 4.3.0 - 10.0 diff --git a/WWTMVC5/Web.config b/WWTMVC5/Web.config index 95628f7c..7ddca09b 100644 --- a/WWTMVC5/Web.config +++ b/WWTMVC5/Web.config @@ -137,6 +137,10 @@ Content-Type: application/x-wt--> + + + + diff --git a/WWTWebSiteOnly.sln b/WWTWebSiteOnly.sln index 9beac5ad..e468c5fe 100644 --- a/WWTWebSiteOnly.sln +++ b/WWTWebSiteOnly.sln @@ -24,6 +24,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WWTWebservices.Azure", "WWTWebservices.Azure\WWTWebservices.Azure.csproj", "{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -46,6 +48,10 @@ Global {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Debug|Any CPU.Build.0 = Debug|Any CPU {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.ActiveCfg = Release|Any CPU {31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.Build.0 = Release|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/WWTWebservices.Azure/AzurePlateTilePyramid.cs b/WWTWebservices.Azure/AzurePlateTilePyramid.cs new file mode 100644 index 00000000..d6724062 --- /dev/null +++ b/WWTWebservices.Azure/AzurePlateTilePyramid.cs @@ -0,0 +1,35 @@ +using Azure.Core; +using Azure.Storage.Blobs; +using System; +using System.Collections.Concurrent; +using System.IO; + +namespace WWTWebservices.Azure +{ + public class AzurePlateTilePyramid : IPlateTilePyramid + { + private readonly BlobServiceClient _service; + private readonly ConcurrentDictionary _containers; + + public AzurePlateTilePyramid(string storageUri, TokenCredential credentials) + { + _service = new BlobServiceClient(new Uri(storageUri), credentials); + _containers = new ConcurrentDictionary(); + } + + public Stream GetStream(string plateName, int level, int x, int y) + { + var container = _containers.GetOrAdd(plateName, p => + { + var name = Path.GetFileNameWithoutExtension(p).ToLowerInvariant(); + + return _service.GetBlobContainerClient(name); + }); + + var client = container.GetBlobClient($"L{level}X{x}Y{y}.png"); + var download = client.Download(); + + return download.Value.Content; + } + } +} diff --git a/WWTWebservices.Azure/WWTWebservices.Azure.csproj b/WWTWebservices.Azure/WWTWebservices.Azure.csproj new file mode 100644 index 00000000..dbaa6343 --- /dev/null +++ b/WWTWebservices.Azure/WWTWebservices.Azure.csproj @@ -0,0 +1,19 @@ + + + + net48 + + + + + + + + + + + + + + + diff --git a/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs b/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs new file mode 100644 index 00000000..97159e39 --- /dev/null +++ b/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs @@ -0,0 +1,69 @@ +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Configuration; +using System.IO; +using System.Text.RegularExpressions; + +namespace WWTWebservices +{ + public class ConfigurationManagerFilePlateTilePyramid : IPlateTilePyramid + { + private readonly RegexDictionary _plateNamePath; + + public ConfigurationManagerFilePlateTilePyramid() + { + _plateNamePath = new RegexDictionary + { + { @"dssterrapixel\.plate", "WWTTilesDir" }, + { @"DSSpngL5to12_x(\d+)_y(\d+)\.plate", "DssTerapixelDir" }, + }; + } + + public Stream GetStream(string plateName, int level, int x, int y) + { + var path = _plateNamePath.GetPath(plateName); + + return PlateTilePyramid.GetFileStream(Path.Combine(path, plateName), level, x, y); + } + + private class RegexDictionary : IEnumerable + { + private readonly Dictionary _regex = new Dictionary(); + private readonly ConcurrentDictionary _strings = new ConcurrentDictionary(); + + public void Add(string pattern, string configName) + { + var path = ConfigurationManager.AppSettings[configName]; + + _regex.Add(new Regex(pattern, RegexOptions.Compiled), path); + } + + IEnumerator IEnumerable.GetEnumerator() => _strings.GetEnumerator(); + + /// + /// Finds the directory of a platefile. + /// + /// The platefile name + /// The directory the platefile is in + /// + /// This uses a list of regex entries that match a prospective platefile to + /// its known directory. Once identified, it is cached so no more regex + /// matches (which are potentially very expensive) must be performed to + /// identify its path. + /// + public string GetPath(string plateFile) => _strings.GetOrAdd(plateFile, p => + { + foreach (var t in _regex) + { + if (t.Key.IsMatch(plateFile)) + { + return t.Value; + } + } + + return null; + }); + } + } +} diff --git a/WWTWebservices/IPlateTilePyramid.cs b/WWTWebservices/IPlateTilePyramid.cs new file mode 100644 index 00000000..c0515e70 --- /dev/null +++ b/WWTWebservices/IPlateTilePyramid.cs @@ -0,0 +1,9 @@ +using System.IO; + +namespace WWTWebservices +{ + public interface IPlateTilePyramid + { + Stream GetStream(string plateName, int level, int x, int y); + } +} diff --git a/WWTWebservices/PlateTilePyramid.cs b/WWTWebservices/PlateTilePyramid.cs index 62322af3..b29d4949 100644 --- a/WWTWebservices/PlateTilePyramid.cs +++ b/WWTWebservices/PlateTilePyramid.cs @@ -238,14 +238,13 @@ public Stream GetFileStream(int level, int x, int y) static public Stream GetFileStream(string filename, int level, int x, int y) { uint offset = GetFileIndexOffset(level, x, y); - uint length; uint start; MemoryStream ms = null; using (FileStream f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { f.Seek(offset, SeekOrigin.Begin); - start = GetNodeInfo(f, offset, out length); + start = GetNodeInfo(f, offset, out var length); byte[] buffer = new byte[length]; f.Seek(start, SeekOrigin.Begin); diff --git a/WWTWebservices/WWTWebservices.csproj b/WWTWebservices/WWTWebservices.csproj index 6e56e41d..b0aa1a75 100644 --- a/WWTWebservices/WWTWebservices.csproj +++ b/WWTWebservices/WWTWebservices.csproj @@ -1,6 +1,6 @@  - net45 + net48 Library false true diff --git a/src/WWT.Providers/Providers/DSSProvider.cs b/src/WWT.Providers/Providers/DSSProvider.cs index f464a488..99f36c50 100644 --- a/src/WWT.Providers/Providers/DSSProvider.cs +++ b/src/WWT.Providers/Providers/DSSProvider.cs @@ -1,5 +1,4 @@ using System; -using System.Configuration; using System.IO; using WWTWebservices; @@ -7,64 +6,53 @@ namespace WWT.Providers { public class DSSProvider : RequestProvider { - public override void Run(WwtContext context) + private readonly IPlateTilePyramid _plateTile; + + public DSSProvider(IPlateTilePyramid plateTile) { - string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"]; - string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"]; + _plateTile = plateTile; + } - string query = context.Request.Params["Q"]; - string[] values = query.Split(','); - int level = Convert.ToInt32(values[0]); - int tileX = Convert.ToInt32(values[1]); - int tileY = Convert.ToInt32(values[2]); + public override void Run(WwtContext context) + { + var (level, tileX, tileY) = context.GetLevelAndCoordinates(); - int octsetlevel = level; - string filename; + using (var stream = GetStream(level, tileX, tileY)) + { + if (stream is null) + { + context.Response.Write("No image"); + context.Response.Close(); + } + context.Response.ContentType = "image/png"; + stream.CopyTo(context.Response.OutputStream); + context.Response.Flush(); + context.Response.End(); + } + } + private Stream GetStream(int level, int tileX, int tileY) + { if (level > 12) { - context.Response.Write("No image"); - context.Response.Close(); - return; + return null; } - - if (level < 8) + else if (level < 8) { - context.Response.ContentType = "image/png"; - Stream s = PlateTilePyramid.GetFileStream(wwtTilesDir + "\\dssterrapixel.plate", level, tileX, tileY); - int length = (int)s.Length; - byte[] data = new byte[length]; - s.Read(data, 0, length); - context.Response.OutputStream.Write(data, 0, length); - context.Response.Flush(); - context.Response.End(); - return; + return _plateTile.GetStream("dssterrapixel.plate", level, tileX, tileY); } else { - int L = level; - int X = tileX; - int Y = tileY; - string mime = "png"; - int powLev5Diff = (int)Math.Pow(2, L - 5); - int X32 = X / powLev5Diff; - int Y32 = Y / powLev5Diff; - filename = string.Format(dssTerapixelDir + @"\DSS{0}L5to12_x{1}_y{2}.plate", mime, X32, Y32); + int L5 = level - 5; + int powLev5Diff = (int)Math.Pow(2, L5); + int X32 = tileX / powLev5Diff; + int Y32 = tileY / powLev5Diff; - int L5 = L - 5; - int X5 = X % powLev5Diff; - int Y5 = Y % powLev5Diff; - context.Response.ContentType = "image/png"; - Stream s = PlateTilePyramid.GetFileStream(filename, L5, X5, Y5); - int length = (int)s.Length; - byte[] data = new byte[length]; - s.Read(data, 0, length); - context.Response.OutputStream.Write(data, 0, length); - context.Response.Flush(); - context.Response.End(); - return; + int X5 = tileX % powLev5Diff; + int Y5 = tileY % powLev5Diff; + return _plateTile.GetStream($"DSSpngL5to12_x{X32}_y{Y32}.plate", L5, X5, Y5); } } } diff --git a/src/WWT.Providers/QueryRequestProvider.cs b/src/WWT.Providers/QueryRequestProvider.cs new file mode 100644 index 00000000..f48c6ac2 --- /dev/null +++ b/src/WWT.Providers/QueryRequestProvider.cs @@ -0,0 +1,18 @@ +using System; + +namespace WWT.Providers +{ + internal static class WwtContextExtensions + { + public static (int level, int tileX, int tileY) GetLevelAndCoordinates(this WwtContext context) + { + string query = context.Request.Params["Q"]; + string[] values = query.Split(','); + int level = Convert.ToInt32(values[0]); + int tileX = Convert.ToInt32(values[1]); + int tileY = Convert.ToInt32(values[2]); + + return (level, tileX, tileY); + } + } +}