Abstract file access pattern for DSS provider

This is an intial change to abstract file access using IPlateTilePyramid. This interface provides a single method that will take a plate file name along with the desired level and coordinates and will retrieve the associated image. This change includes two implementations of this: - ConfigurationManagerFilePlateTilePyramid allows for a request to a given plate file and returns a stream for the level and coordinates specified. - AzurePlateTilePyramid retrieves the file from Azure blob storage This new access method has been incorporated into the DSS.aspx page and will surface the content via Azure or local storage via a configuration flag (requires a restart of the service).
WorldWideTelescope · Sep 21, 2020 · 18d4f4c · 18d4f4c
1 parent 66d14e2
commit 18d4f4c
Show file tree

Hide file tree

Showing 11 changed files with 147 additions and 47 deletions.
diff --git a/WWTMVC5/App_Start/UnityConfig.cs b/WWTMVC5/App_Start/UnityConfig.cs
@@ -1,7 +1,12 @@
 using System;
+using System.Configuration;
 using System.Linq;
+using Azure.Core;
+using Azure.Identity;
 using Microsoft.Practices.Unity;
 using WWT.Providers;
+using WWTWebservices;
+using WWTWebservices.Azure;
 
 namespace WWTMVC5
 {
@@ -34,6 +39,7 @@ public static IUnityContainer GetConfiguredContainer()
         public static void RegisterTypes(IUnityContainer container)
         {
             RegisterRequestProviders(container);
+            RegisterPlateFileProvider(container);
 
             // NOTE: To load from web.config uncomment the line below. Make sure to add a Microsoft.Practices.Unity.Configuration to the using statements.
             // container.LoadConfiguration();
@@ -52,5 +58,17 @@ private static void RegisterRequestProviders(IUnityContainer container)
                 container.RegisterType(type);
             }
         }
+
+        private static void RegisterPlateFileProvider(IUnityContainer container)
+        {
+            if (ConfigReader<bool>.GetSetting("UseAzurePlateFiles"))
+            {
+                container.RegisterInstance<IPlateTilePyramid>(new AzurePlateTilePyramid(ConfigurationManager.AppSettings["AzurePlateFileContainer"], new DefaultAzureCredential()));
+            }
+            else
+            {
+                container.RegisterType<IPlateTilePyramid, ConfigurationManagerFilePlateTilePyramid>(new ContainerControlledLifetimeManager());
+            }
+        }
     }
 }
diff --git a/WWTMVC5/WWTMVC5.csproj b/WWTMVC5/WWTMVC5.csproj
@@ -1377,6 +1377,10 @@
       <Project>{ee4a3106-572b-4ef4-9ab5-a22643309a57}</Project>
       <Name>WWT.Providers</Name>
     </ProjectReference>
+    <ProjectReference Include="..\WWTWebservices.Azure\WWTWebservices.Azure.csproj">
+      <Project>{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}</Project>
+      <Name>WWTWebservices.Azure</Name>
+    </ProjectReference>
     <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj">
       <Project>{1cf4d986-51ad-43f8-a84a-38b6ecba2172}</Project>
       <Name>WWTWebservices</Name>
@@ -1443,9 +1447,6 @@
     <PackageReference Include="Microsoft.IdentityModel">
       <Version>6.1.7600.16394</Version>
     </PackageReference>
-    <PackageReference Include="Microsoft.WindowsAzure.ConfigurationManager">
-      <Version>3.2.3</Version>
-    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>10.0.3</Version>
     </PackageReference>
@@ -1461,9 +1462,6 @@
     <PackageReference Include="WebGrease">
       <Version>1.6.0</Version>
     </PackageReference>
-    <PackageReference Include="WindowsAzure.Storage">
-      <Version>4.3.0</Version>
-    </PackageReference>
   </ItemGroup>
   <PropertyGroup>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

diff --git a/WWTMVC5/Web.config b/WWTMVC5/Web.config
@@ -137,6 +137,10 @@ Content-Type: application/x-wt-->
     <add key="ResourcesVersion" value="5.3.65"/>
     <add key="HomePageCacheTimeout" value="5"/>
 
+    <!-- Azure storage access for plate files-->
+    <add key="UseAzurePlateFiles" value="true"/>
+    <add key="AzurePlateFileContainer" value="https://127.0.0.1:10000/devstoreaccount1"/>
+
     <!-- Overridden with Configuration Builders -->
     <add key="TourCache" value=""/>
     <add key="WWTTilesDir" value=""/>

diff --git a/WWTWebSiteOnly.sln b/WWTWebSiteOnly.sln
@@ -24,6 +24,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		README.md = README.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WWTWebservices.Azure", "WWTWebservices.Azure\WWTWebservices.Azure.csproj", "{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -46,6 +48,10 @@ Global
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{31DC6DC8-AF43-41AC-B9B7-7E77E6CD8950}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FB9956F9-B0FF-4E45-BAA5-4180DA8A12A9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/WWTWebservices.Azure/AzurePlateTilePyramid.cs b/WWTWebservices.Azure/AzurePlateTilePyramid.cs
@@ -0,0 +1,35 @@
+using Azure.Core;
+using Azure.Storage.Blobs;
+using System;
+using System.Collections.Concurrent;
+using System.IO;
+
+namespace WWTWebservices.Azure
+{
+    public class AzurePlateTilePyramid : IPlateTilePyramid
+    {
+        private readonly BlobServiceClient _service;
+        private readonly ConcurrentDictionary<string, BlobContainerClient> _containers;
+
+        public AzurePlateTilePyramid(string storageUri, TokenCredential credentials)
+        {
+            _service = new BlobServiceClient(new Uri(storageUri), credentials);
+            _containers = new ConcurrentDictionary<string, BlobContainerClient>();
+        }
+
+        public Stream GetStream(string pathPrefix, string plateName, int level, int x, int y)
+        {
+            var container = _containers.GetOrAdd(plateName, p =>
+            {
+                var name = Path.GetFileNameWithoutExtension(p).ToLowerInvariant();
+
+                return _service.GetBlobContainerClient(name);
+            });
+
+            var client = container.GetBlobClient($"L{level}X{x}Y{y}.png");
+            var download = client.Download();
+
+            return download.Value.Content;
+        }
+    }
+}
diff --git a/WWTWebservices.Azure/WWTWebservices.Azure.csproj b/WWTWebservices.Azure/WWTWebservices.Azure.csproj
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net48</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.6.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\WWTWebservices\WWTWebservices.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Reference Include="System.Configuration" />
+  </ItemGroup>
+
+</Project>
diff --git a/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs b/WWTWebservices/ConfigurationManagerFilePlateTilePyramid.cs
@@ -0,0 +1,17 @@
+using System.IO;
+
+namespace WWTWebservices
+{
+    public class ConfigurationManagerFilePlateTilePyramid : IPlateTilePyramid
+    {
+        public Stream GetStream(string pathPrefix, string plateName, int level, int x, int y)
+        {
+            if (string.IsNullOrEmpty(pathPrefix))
+            {
+                throw new System.ArgumentException($"'{nameof(pathPrefix)}' cannot be null or empty", nameof(pathPrefix));
+            }
+
+            return PlateTilePyramid.GetFileStream(Path.Combine(pathPrefix, plateName), level, x, y);
+        }
+    }
+}
diff --git a/WWTWebservices/IPlateTilePyramid.cs b/WWTWebservices/IPlateTilePyramid.cs
@@ -0,0 +1,9 @@
+using System.IO;
+
+namespace WWTWebservices
+{
+    public interface IPlateTilePyramid
+    {
+        Stream GetStream(string pathPrefix, string plateName, int level, int x, int y);
+    }
+}
diff --git a/WWTWebservices/PlateTilePyramid.cs b/WWTWebservices/PlateTilePyramid.cs
@@ -238,14 +238,13 @@ public Stream GetFileStream(int level, int x, int y)
         static public Stream GetFileStream(string filename, int level, int x, int y)
         {
             uint offset = GetFileIndexOffset(level, x, y);
-            uint length;
             uint start;
 
             MemoryStream ms = null;
             using (FileStream f = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
             {
                 f.Seek(offset, SeekOrigin.Begin);
-                start = GetNodeInfo(f, offset, out length);
+                start = GetNodeInfo(f, offset, out var length);
 
                 byte[] buffer = new byte[length];
                 f.Seek(start, SeekOrigin.Begin);

diff --git a/WWTWebservices/WWTWebservices.csproj b/WWTWebservices/WWTWebservices.csproj
@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
   <PropertyGroup>
-    <TargetFramework>net45</TargetFramework>
+    <TargetFramework>net48</TargetFramework>
     <OutputType>Library</OutputType>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <UseWindowsForms>true</UseWindowsForms>

diff --git a/src/WWT.Providers/Providers/DSSProvider.cs b/src/WWT.Providers/Providers/DSSProvider.cs
@@ -1,70 +1,65 @@
 using System;
 using System.Configuration;
-using System.IO;
 using WWTWebservices;
 
 namespace WWT.Providers
 {
     public class DSSProvider : RequestProvider
     {
-        public override void Run(WwtContext context)
+        private readonly IPlateTilePyramid _plateTile;
+
+        public DSSProvider(IPlateTilePyramid plateTile)
         {
-            string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"];
-            string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"];
+            _plateTile = plateTile;
+        }
 
+        public override void Run(WwtContext context)
+        {
             string query = context.Request.Params["Q"];
             string[] values = query.Split(',');
             int level = Convert.ToInt32(values[0]);
             int tileX = Convert.ToInt32(values[1]);
             int tileY = Convert.ToInt32(values[2]);
 
-            int octsetlevel = level;
-            string filename;
-
-
             if (level > 12)
             {
                 context.Response.Write("No image");
                 context.Response.Close();
-                return;
             }
-
-            if (level < 8)
+            else if (level < 8)
             {
+                string wwtTilesDir = ConfigurationManager.AppSettings["WWTTilesDir"];
+
                 context.Response.ContentType = "image/png";
-                Stream s = PlateTilePyramid.GetFileStream(wwtTilesDir + "\\dssterrapixel.plate", level, tileX, tileY);
-                int length = (int)s.Length;
-                byte[] data = new byte[length];
-                s.Read(data, 0, length);
-                context.Response.OutputStream.Write(data, 0, length);
-                context.Response.Flush();
-                context.Response.End();
-                return;
+
+                using (var s = _plateTile.GetStream(wwtTilesDir, "dssterrapixel.plate", level, tileX, tileY))
+                {
+                    s.CopyTo(context.Response.OutputStream);
+                    context.Response.Flush();
+                    context.Response.End();
+                }
             }
             else
             {
-                int L = level;
-                int X = tileX;
-                int Y = tileY;
-                string mime = "png";
-                int powLev5Diff = (int)Math.Pow(2, L - 5);
-                int X32 = X / powLev5Diff;
-                int Y32 = Y / powLev5Diff;
-                filename = string.Format(dssTerapixelDir + @"\DSS{0}L5to12_x{1}_y{2}.plate", mime, X32, Y32);
+                int powLev5Diff = (int)Math.Pow(2, level - 5);
+                int X32 = tileX / powLev5Diff;
+                int Y32 = tileY / powLev5Diff;
+
+                int L5 = level - 5;
+                int X5 = tileX % powLev5Diff;
+                int Y5 = tileY % powLev5Diff;
 
-                int L5 = L - 5;
-                int X5 = X % powLev5Diff;
-                int Y5 = Y % powLev5Diff;
                 context.Response.ContentType = "image/png";
-                Stream s = PlateTilePyramid.GetFileStream(filename, L5, X5, Y5);
-                int length = (int)s.Length;
-                byte[] data = new byte[length];
-                s.Read(data, 0, length);
-                context.Response.OutputStream.Write(data, 0, length);
-                context.Response.Flush();
-                context.Response.End();
-                return;
 
+                string dssTerapixelDir = ConfigurationManager.AppSettings["DssTerapixelDir"];
+                string filename = $"DSSpngL5to12_x{X32}_y{Y32}.plate";
+
+                using (var s = _plateTile.GetStream(dssTerapixelDir, filename, L5, X5, Y5))
+                {
+                    s.CopyTo(context.Response.OutputStream);
+                    context.Response.Flush();
+                    context.Response.End();
+                }
             }
         }
     }