Abstract file access pattern for DSS provider

This is an intial change to abstract file access using
IPlateTilePyramid. This interface provides a single method that will
take a plate file name along with the desired level and coordinates and
will retrieve the associated image.

This change includes two implementations of this:

- ConfigurationManagerFilePlateTilePyramid allows for a request to a
  given plate file and returns a stream for the level and coordinates
  specified.
- AzurePlateTilePyramid retrieves the file from Azure blob storage

This new access method has been incorporated into the DSS.aspx page and
will surface the content via Azure or local storage via a configuration
flag (requires a restart of the service).

WWTMVC5/App_Start/UnityConfig.cs

@@ -1,7 +1,12 @@
 using System;
+using System.Configuration;
 using System.Linq;
+using Azure.Core;
+using Azure.Identity;
 using Microsoft.Practices.Unity;
 using WWT.Providers;
+using WWTWebservices;
+using WWTWebservices.Azure;
 
 namespace WWTMVC5
 {
@@ -34,6 +39,7 @@ public static IUnityContainer GetConfiguredContainer()
         public static void RegisterTypes(IUnityContainer container)
         {
             RegisterRequestProviders(container);
+            RegisterPlateFileProvider(container);
 
             // NOTE: To load from web.config uncomment the line below. Make sure to add a Microsoft.Practices.Unity.Configuration to the using statements.
             // container.LoadConfiguration();
@@ -52,5 +58,17 @@ private static void RegisterRequestProviders(IUnityContainer container)
                 container.RegisterType(type);
             }
         }
+
+        private static void RegisterPlateFileProvider(IUnityContainer container)
+        {
+            if (ConfigReader<bool>.GetSetting("UseAzurePlateFiles"))
+            {
+                container.RegisterInstance<IPlateTilePyramid>(new AzurePlateTilePyramid(ConfigurationManager.AppSettings["AzurePlateFileContainer"], new DefaultAzureCredential()));
+            }
+            else
+            {
+                container.RegisterType<IPlateTilePyramid, ConfigurationManagerFilePlateTilePyramid>(new ContainerControlledLifetimeManager());
+            }
+        }
     }
 }

WWTMVC5/WWTMVC5.csproj

@@ -1377,6 +1377,10 @@
       <Project>{ee4a3106-572b-4ef4-9ab5-a22643309a57}</Project>
       <Name>WWT.Providers</Name>
     </ProjectReference>
+    <ProjectReference Include="..\WWTWebservices.Azure\WWTWebservices.Azure.csproj">
+      <Project>{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}</Project>
+      <Name>WWTWebservices.Azure</Name>
+    </ProjectReference>
     <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj">
       <Project>{1cf4d986-51ad-43f8-a84a-38b6ecba2172}</Project>
       <Name>WWTWebservices</Name>
@@ -1443,9 +1447,6 @@
     <PackageReference Include="Microsoft.IdentityModel">
       <Version>6.1.7600.16394</Version>
     </PackageReference>
-    <PackageReference Include="Microsoft.WindowsAzure.ConfigurationManager">
-      <Version>3.2.3</Version>
-    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>10.0.3</Version>
     </PackageReference>
@@ -1461,9 +1462,6 @@
     <PackageReference Include="WebGrease">
       <Version>1.6.0</Version>
     </PackageReference>
-    <PackageReference Include="WindowsAzure.Storage">
-      <Version>4.3.0</Version>
-    </PackageReference>
   </ItemGroup>
   <PropertyGroup>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

WWTMVC5/Web.config

@@ -137,6 +137,10 @@ Content-Type: application/x-wt-->
     <add key="ResourcesVersion" value="5.3.65"/>
     <add key="HomePageCacheTimeout" value="5"/>
 
+    <!-- Azure storage access for plate files-->
+    <add key="UseAzurePlateFiles" value="true"/>
+    <add key="AzurePlateFileContainer" value="https://127.0.0.1:10000/devstoreaccount1"/>
+
     <!-- Overridden with Configuration Builders -->
     <add key="TourCache" value=""/>
     <add key="WWTTilesDir" value=""/>

WWTWebSiteOnly.sln

@@ -24,6 +24,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		README.md = README.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WWTWebservices.Azure", "WWTWebservices.Azure\WWTWebservices.Azure.csproj", "{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -46,6 +48,10 @@ Global
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

WWTWebservices.Azure/AzurePlateTilePyramid.cs

@@ -0,0 +1,35 @@
+using Azure.Core;
+using Azure.Storage.Blobs;
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+
+namespace WWTWebservices.Azure
+{
+    public class AzurePlateTilePyramid : IPlateTilePyramid
+    {
+        private readonly BlobServiceClient _service;
+        private readonly ConcurrentDictionary<string, BlobContainerClient> _containers;
+
+        public AzurePlateTilePyramid(string storageUri, TokenCredential credentials)
+        {
+            _service = new BlobServiceClient(new Uri(storageUri), credentials);
+            _containers = new ConcurrentDictionary<string, BlobContainerClient>();
+        }
+
+        public Stream GetStream(string plateName, int level, int x, int y)
+        {
+            var container = _containers.GetOrAdd(plateName, p =>
+            {
+                var name = Path.GetFileNameWithoutExtension(p).ToLowerInvariant();
+
+                return _service.GetBlobContainerClient(name);
+            });
+
+            var client = container.GetBlobClient($"L{level}X{x}Y{y}.png");
+            var download = client.Download();
+
+            return download.Value.Content;
+        }
+    }
+}

WWTWebservices.Azure/WWTWebservices.Azure.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net48</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.6.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="System.Configuration" />
+  </ItemGroup>
+
+</Project>

WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs

@@ -0,0 +1,69 @@
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Configuration;
+using System.IO;
+using System.Text.RegularExpressions;
+
+namespace WWTWebservices
+{
+    public class ConfigurationManagerFilePlateTilePyramid : IPlateTilePyramid
+    {
+        private readonly RegexDictionary _plateNamePath;
+
+        public ConfigurationManagerFilePlateTilePyramid()
+        {
+            _plateNamePath = new RegexDictionary
+            {
+                { @"dssterrapixel\.plate", "WWTTilesDir" },
+                { @"DSSpngL5to12_x(\d+)_y(\d+)\.plate", "DssTerapixelDir" },
+            };
+        }
+
+        public Stream GetStream(string plateName, int level, int x, int y)
+        {
+            var path = _plateNamePath.GetPath(plateName);
+
+            return PlateTilePyramid.GetFileStream(Path.Combine(path, plateName), level, x, y);
+        }
+
+        private class RegexDictionary : IEnumerable
+        {
+            private readonly Dictionary<Regex, string> _regex = new Dictionary<Regex, string>();
+            private readonly ConcurrentDictionary<string, string> _strings = new ConcurrentDictionary<string, string>();
+
+            public void Add(string pattern, string configName)
+            {
+                var path = ConfigurationManager.AppSettings[configName];
+
+                _regex.Add(new Regex(pattern, RegexOptions.Compiled), path);
+            }
+
+            IEnumerator IEnumerable.GetEnumerator() => _strings.GetEnumerator();
+
+            /// <summary>
+            /// Finds the directory of a platefile.
+            /// </summary>
+            /// <param name="plateFile">The platefile name</param>
+            /// <returns>The directory the platefile is in</returns>
+            /// <remarks>
+            /// This uses a list of regex entries that match a prospective platefile to
+            /// its known directory. Once identified, it is cached so no more regex
+            /// matches (which are potentially very expensive) must be performed to
+            /// identify its path.
+            /// </remarks>
+            public string GetPath(string plateFile) => _strings.GetOrAdd(plateFile, p =>
+            {
+                foreach (var t in _regex)
+                {
+                    if (t.Key.IsMatch(plateFile))
+                    {
+                        return t.Value;
+                    }
+                }
+
+                return null;
+            });
+        }
+    }
+}

WWTWebservices/IPlateTilePyramid.cs

@@ -0,0 +1,9 @@
+using System.IO;
+
+namespace WWTWebservices
+{
+    public interface IPlateTilePyramid
+    {
+        Stream GetStream(string plateName, int level, int x, int y);
+    }
+}

WWTWebservices/PlateTilePyramid.cs

@@ -238,14 +238,13 @@ public Stream GetFileStream(int level, int x, int y)
         static public Stream GetFileStream(string filename, int level, int x, int y)
         {
             uint offset = GetFileIndexOffset(level, x, y);
-            uint length;
             uint start;
 
             MemoryStream ms = null;
             using (FileStream f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
             {
                 f.Seek(offset, SeekOrigin.Begin);
-                start = GetNodeInfo(f, offset, out length);
+                start = GetNodeInfo(f, offset, out var length);
 
                 byte[] buffer = new byte[length];
                 f.Seek(start, SeekOrigin.Begin);

WWTWebservices/WWTWebservices.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
   <PropertyGroup>
-    <TargetFramework>net45</TargetFramework>
+    <TargetFramework>net48</TargetFramework>
     <OutputType>Library</OutputType>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <UseWindowsForms>true</UseWindowsForms>

src/WWT.Providers/Providers/DSSProvider.cs

@@ -1,70 +1,58 @@
 using System;
-using System.Configuration;
 using System.IO;
 using WWTWebservices;
 
 namespace WWT.Providers
 {
     public class DSSProvider : RequestProvider
     {
-        public override void Run(WwtContext context)
+        private readonly IPlateTilePyramid _plateTile;
+
+        public DSSProvider(IPlateTilePyramid plateTile)
         {
-            string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"];
-            string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"];
+            _plateTile = plateTile;
+        }
 
-            string query = context.Request.Params["Q"];
-            string[] values = query.Split(',');
-            int level = Convert.ToInt32(values[0]);
-            int tileX = Convert.ToInt32(values[1]);
-            int tileY = Convert.ToInt32(values[2]);
+        public override void Run(WwtContext context)
+        {
+            var (level, tileX, tileY) = context.GetLevelAndCoordinates();
 
-            int octsetlevel = level;
-            string filename;
+            using (var stream = GetStream(level, tileX, tileY))
+            {
+                if (stream is null)
+                {
+                    context.Response.Write("No image");
+                    context.Response.Close();
+                }
 
+                context.Response.ContentType = "image/png";
+                stream.CopyTo(context.Response.OutputStream);
+                context.Response.Flush();
+                context.Response.End();
+            }
+        }
 
+        private Stream GetStream(int level, int tileX, int tileY)
+        {
             if (level > 12)
             {
-                context.Response.Write("No image");
-                context.Response.Close();
-                return;
+                return null;
             }
-
-            if (level < 8)
+            else if (level < 8)
             {
-                context.Response.ContentType = "image/png";
-                Stream s = PlateTilePyramid.GetFileStream(wwtTilesDir + "\\dssterrapixel.plate", level, tileX, tileY);
-                int length = (int)s.Length;
-                byte[] data = new byte[length];
-                s.Read(data, 0, length);
-                context.Response.OutputStream.Write(data, 0, length);
-                context.Response.Flush();
-                context.Response.End();
-                return;
+                return _plateTile.GetStream("dssterrapixel.plate", level, tileX, tileY);
             }
             else
             {
-                int L = level;
-                int X = tileX;
-                int Y = tileY;
-                string mime = "png";
-                int powLev5Diff = (int)Math.Pow(2, L - 5);
-                int X32 = X / powLev5Diff;
-                int Y32 = Y / powLev5Diff;
-                filename = string.Format(dssTerapixelDir + @"\DSS{0}L5to12_x{1}_y{2}.plate", mime, X32, Y32);
+                int L5 = level - 5;
+                int powLev5Diff = (int)Math.Pow(2, L5);
+                int X32 = tileX / powLev5Diff;
+                int Y32 = tileY / powLev5Diff;
 
-                int L5 = L - 5;
-                int X5 = X % powLev5Diff;
-                int Y5 = Y % powLev5Diff;
-                context.Response.ContentType = "image/png";
-                Stream s = PlateTilePyramid.GetFileStream(filename, L5, X5, Y5);
-                int length = (int)s.Length;
-                byte[] data = new byte[length];
-                s.Read(data, 0, length);
-                context.Response.OutputStream.Write(data, 0, length);
-                context.Response.Flush();
-                context.Response.End();
-                return;
+                int X5 = tileX % powLev5Diff;
+                int Y5 = tileY % powLev5Diff;
 
+                return _plateTile.GetStream($"DSSpngL5to12_x{X32}_y{Y32}.plate", L5, X5, Y5);
             }
         }
     }

src/WWT.Providers/QueryRequestProvider.cs

@@ -0,0 +1,18 @@
+using System;
+
+namespace WWT.Providers
+{
+    internal static class WwtContextExtensions
+    {
+        public static (int level, int tileX, int tileY) GetLevelAndCoordinates(this WwtContext context)
+        {
+            string query = context.Request.Params["Q"];
+            string[] values = query.Split(',');
+            int level = Convert.ToInt32(values[0]);
+            int tileX = Convert.ToInt32(values[1]);
+            int tileY = Convert.ToInt32(values[2]);
+
+            return (level, tileX, tileY);
+        }
+    }
+}