Improvements to performance of batch processing

src/Lithnet.Pan.RAProxy.Setup/Components.wxs

@@ -27,6 +27,8 @@
           <util:PerformanceCounter Name="Requests in queue" Type="numberOfItems32"  Help="Shows the number of requests in the queue that are waiting to be sent to the firewall"/>
           <util:PerformanceCounter Name="Requests received / second" Type="rateOfCountsPerSecond32" Help="Shows the number of account requests receieved per second"/>
           <util:PerformanceCounter Name="Requests sent / second" Type="rateOfCountsPerSecond32" Help="Shows the number of account requests sent per second"/>
+          <util:PerformanceCounter Name="Failed mappings / second" Type="rateOfCountsPerSecond32" Help="Shows the number of failed user-id mappings per second"/>
+
         </util:PerformanceCategory>
       </Component>
 

src/Lithnet.Pan.RAProxy/App.config

@@ -5,8 +5,8 @@
   </configSections>
 
   <ra-proxy-config debug-enabled="true">
-    <pan-api-endpoints disable-certificate-validation="true">
-      <pan-api-endpoint url="https://test-fw/api/" api-key="secret" />
+    <pan-api-endpoints disable-certificate-validation="true" batch-size="200" batch-wait="50">
+      <pan-api-endpoint url="https://test-fw/api/" api-key="LUFRPT1Oa1lhZVlvT2JTckhTY2pRcVpiOStnRkF3OEk9WlR5NENPNEE3MGVFcVRtQ1JLTldSWGtkdC9EcVhRc3VDTW41b0R3eGlHVT0" />
     </pan-api-endpoints>
 
     <radius-servers>

src/Lithnet.Pan.RAProxy/ConfigSections/Config.cs

@@ -72,6 +72,11 @@ public static void Failover()
 
         public static bool DebuggingEnabled => Config.section.DebuggingEnabled;
 
+        public static int BatchSize => Config.section.PanApi.BatchSize;
+
+        public static int BatchWait => Config.section.PanApi.BatchWait;
+
+
         public static string GetSecretForIP(IPAddress address)
         {
             string addressStringForm = address.ToString();

src/Lithnet.Pan.RAProxy/ConfigSections/PanApiCollection.cs

@@ -25,6 +25,34 @@ public bool DisableCertificateValidation
             }
         }
 
+        [ConfigurationProperty("batch-size", IsRequired = false, DefaultValue = 150)]
+        public int BatchSize
+        {
+            get
+            {
+                return (int)base["batch-size"];
+            }
+
+            set
+            {
+                base["batch-size"] = value;
+            }
+        }
+
+        [ConfigurationProperty("batch-wait", IsRequired = false, DefaultValue = 150)]
+        public int BatchWait
+        {
+            get
+            {
+                return (int)base["batch-wait"];
+            }
+
+            set
+            {
+                base["batch-wait"] = value;
+            }
+        }
+
         protected override object GetElementKey(ConfigurationElement element)
         {
             return ((PanApiEndpoint)element).ApiUri;

src/Lithnet.Pan.RAProxy/Exceptions/AggregateUserMappingException.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lithnet.Pan.RAProxy
+{
+    public class AggregateUserMappingException : AggregateException
+    {
+        public AggregateUserMappingException(string message, IEnumerable<Exception> exceptions)
+            : base(message, exceptions)
+        {
+        }
+    }
+}

src/Lithnet.Pan.RAProxy/Exceptions/UserMappingException.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lithnet.Pan.RAProxy
+{
+    public class UserMappingException : PanApiException
+    {
+        public string Username { get; private set; }
+
+        public string IPAddress { get; private set; }
+
+        public UserMappingException(string message, string username, string ipaddress)
+            :base (message)
+        {
+            this.Username = username;
+            this.IPAddress = ipaddress;
+        }
+    }
+}

src/Lithnet.Pan.RAProxy/Lithnet.Pan.RAProxy.csproj

@@ -54,6 +54,8 @@
     <Compile Include="Exceptions\MissingValueException.cs" />
     <Compile Include="Exceptions\InvalidRadiusAttributeException.cs" />
     <Compile Include="Exceptions\UnknownRadiusServerException.cs" />
+    <Compile Include="Exceptions\AggregateUserMappingException.cs" />
+    <Compile Include="Exceptions\UserMappingException.cs" />
     <Compile Include="Extensions.cs" />
     <Compile Include="Exceptions\PanApiException.cs" />
     <Compile Include="Logging.cs" />

src/Lithnet.Pan.RAProxy/Logging.cs

@@ -15,13 +15,17 @@ internal class Logging
         public const int EventIDApiEndpointExceptionWillFailover = 3005;
         public const int EventIDApiEndpointFailover = 3006;
 
+        public const int EventIDApiUserIDMappingLoginFailed= 3007;
+        public const int EventIDApiUserIDMappingLogoutFailed = 3008;
+
 
         // Accounting Errors
         public const int EventIDUnknownRadiusHost = 3101;
         public const int EventIDMissingAttribute = 3102;
 
         // Messaging errors
         public const int EventIDMessageSendFailure = 3201;
+        public const int EventIDInvalidRadiusPacket = 3202;
 
         // Info
         public const int EventIDAccountingRequestRecieved = 4001;
@@ -33,6 +37,9 @@ internal class Logging
 
         public static PerformanceCounter CounterItemsInQueue { get; }
 
+        public static PerformanceCounter CounterFailedMappingsPerSecond { get; }
+
+
         public static void WriteEntry(string message, EventLogEntryType type, int eventID)
         {
             Trace.WriteLine(message);
@@ -76,6 +83,16 @@ static Logging()
                 MachineName = ".",
                 ReadOnly = false
             };
+
+            Logging.CounterFailedMappingsPerSecond = new PerformanceCounter
+            {
+                CategoryName = "PANRAProxy",
+                CounterName = "Failed mappings / second",
+                MachineName = ".",
+                ReadOnly = false
+            };
+
+            Logging.CounterItemsInQueue.RawValue = 0;
         }
     }
 }

src/Lithnet.Pan.RAProxy/PanApi/Message.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Text;
 using System.IO;
 using System.Net;
@@ -36,49 +37,116 @@ public void Send()
                 d.LoadXml(response);
 
                 XmlNode status = d.SelectSingleNode("/response/@status");
+                List<Exception> exceptions = new List<Exception>();
 
-                if (status != null)
+                if (status == null)
                 {
-                    if (status.InnerText == "success")
+                    Logging.WriteEntry($"The API called failed with an unknown result\n{response}", EventLogEntryType.Error, Logging.EventIDUnknownApiException);
+                    throw new PanApiException($"The API called failed with an unknown result", response);
+                }
+
+                if (status.InnerText == "success")
+                {
+                    return;
+                }
+
+                if (status.InnerText == "error")
+                {
+                    XmlNodeList xmlNodeList = d.SelectNodes("/response/msg/line/uid-response/payload/logout/entry");
+
+                    if (xmlNodeList != null)
                     {
-                        return;
+                        foreach (UserMappingException ex in Message.GetExceptions(xmlNodeList))
+                        {
+                            exceptions.Add(ex);
+                            Logging.WriteEntry($"The logout user mapping for {ex.Username} with ip {ex.IPAddress} failed with message '{ex.Message}'", EventLogEntryType.Error, Logging.EventIDApiUserIDMappingLogoutFailed);
+                        }
                     }
 
-                    if (status.InnerText == "error")
-                    {
-                        XmlNode message = d.SelectSingleNode("/response/msg/line/uid-response/payload/logout/entry/@message");
+                    xmlNodeList = d.SelectNodes("/response/msg/line/uid-response/payload/login/entry");
 
-                        if (message != null)
+                    if (xmlNodeList != null)
+                    {
+                        foreach (UserMappingException ex in Message.GetExceptions(xmlNodeList))
                         {
-                            if (message.InnerText.Equals("delete mapping failed", StringComparison.InvariantCultureIgnoreCase))
-                            {
-                                if (!Config.DebuggingEnabled)
-                                {
-                                    return;
-                                }
-                            }
+                            exceptions.Add(ex);
+                            Logging.WriteEntry($"The login user mapping for {ex.Username} with ip {ex.IPAddress} failed with message '{ex.Message}'", EventLogEntryType.Error, Logging.EventIDApiUserIDMappingLoginFailed);
                         }
                     }
+                }
 
-                    Logging.WriteEntry($"The API called failed with status {status.InnerText}\n{response}", EventLogEntryType.Error, Logging.EventIDApiException);
-                    throw new PanApiException($"The API called failed with status {status.InnerText}", response);
-
+                if (exceptions.Count == 1)
+                {
+                    throw exceptions[0];
                 }
-                else
+
+                if (exceptions.Count > 1)
                 {
-                    Logging.WriteEntry($"The API called failed with an unknown result\n{response}", EventLogEntryType.Error, Logging.EventIDUnknownApiException);
-                    throw new PanApiException($"The API called failed with an unknown result", response);
+                    throw new AggregateUserMappingException("Multiple user mapping operations failed", exceptions);
                 }
             }
+            catch (AggregateException)
+            {
+                throw;
+            }
+            catch (PanApiException)
+            {
+                throw;
+            }
             catch
             {
-                Logging.WriteEntry($"The API called failed with an unsupported response\n{response}", EventLogEntryType.Error, Logging.EventIDUnknownApiResponse);
-                throw new PanApiException($"The API called failed with an unsupported response", response);
+                Logging.WriteEntry($"An error occurred parsing the API response\n{response}", EventLogEntryType.Error, Logging.EventIDUnknownApiResponse);
+                throw new PanApiException($"An error occurred parsing the API response", response);
             }
         }
 
+
+        private static IList<UserMappingException> GetExceptions(XmlNodeList xmlNodeList)
+        {
+            List<UserMappingException> exceptions = new List<UserMappingException>();
+
+            foreach (XmlNode entry in xmlNodeList)
+            {
+                UserMappingException e = GetException(entry);
+
+                if (e != null)
+                {
+                    exceptions.Add(e);
+                }
+            }
+
+            return exceptions;
+        }
+
+        private static UserMappingException GetException(XmlNode entry)
+        {
+            XmlAttributeCollection attributes = entry.Attributes;
+
+            if (attributes == null)
+            {
+                return null;
+            }
+
+            UserMappingException e = new UserMappingException(
+                attributes["message"]?.InnerText,
+                attributes["name"]?.InnerText,
+                attributes["ip"]?.InnerText);
+
+            if (e.Message.Equals("delete mapping failed", StringComparison.InvariantCultureIgnoreCase))
+            {
+                if (!Config.DebuggingEnabled)
+                {
+                    return null;
+                }
+            }
+
+            Logging.CounterFailedMappingsPerSecond.Increment();
+
+            return e;
+        }
+
         /*\
-         
+
              <response status="error">
                  <msg>
                   <line>
@@ -90,9 +158,9 @@ public void Send()
                             </logout>
                         </payload>
                     </uid-response>
-</line></msg></response>
+        </line></msg></response>
 
- */
+        */
 
         private string Submit()
         {
@@ -131,6 +199,7 @@ private string Submit(PanApiEndpoint ep)
             builder.Query = queryString.ToString();
 
             HttpWebRequest request = this.GetRequestContent(builder.Uri, messageText);
+            request.ServicePoint.Expect100Continue = false;
 
             using (WebResponse response = request.GetResponse())
             {