From 08a9f7daf580250f4c50402ea660e25b76ba370f Mon Sep 17 00:00:00 2001 From: Matthew Jackson <123390480+mjaksn@users.noreply.github.com> Date: Thu, 4 Jan 2024 12:02:24 -0600 Subject: [PATCH] [ODS-6084] Remove XML Lookup Utility (#901) --- ...ucationOrganizationCacheLookupStepTests.cs | 166 -------------- .../LookupPipelineTests.cs | 72 ------ ...MapXmlLookupToGetByExampleJsonStepTests.cs | 210 ------------------ .../XmlToWorkItemsProcessorTests.cs | 184 --------------- .../XmlLookupToResourceProcessorTests.cs | 122 ---------- .../EducationOrganizationIdentityCache.cs | 143 ------------ .../Engine/Factories/XmlIoFactory.cs | 58 ----- .../IEducationOrganizationIdentityCache.cs | 15 -- .../EdFi.LoadTools/Engine/Interfaces.cs | 15 -- .../AvoidDuplicateLookupsStep.cs | 34 --- .../XmlLookupPipeline/ComputeHashStep.cs | 27 --- .../DirectLookupToIdentityMappingStep.cs | 70 ------ .../EducationOrganizationCacheLookupStep.cs | 83 ------- .../IdentifyResourceTypeStep.cs | 34 --- .../MapResourceToIdentityStep.cs | 70 ------ .../MapXmlLookupToGetByExampleJsonStep.cs | 70 ------ .../PerformGetByExampleStep.cs | 90 -------- .../StoreIdentityForWritingStep.cs | 30 --- .../XmlLookupPipelineProcessor.cs | 30 --- .../XmlLookupToResourceProcessor.cs | 202 ----------------- .../XmlLookupPipeline/XmlLookupWorkItem.cs | 37 --- .../XmlToWorkItemsProcessor.cs | 83 ------- .../EdFi.LoadTools/XmlLookupApplication.cs | 95 -------- Utilities/DataLoading/LoadTools.sln | 10 - 24 files changed, 1950 deletions(-) delete mode 100644 Utilities/DataLoading/EdFi.LoadTools.Test/EducationOrganizationCacheLookupStepTests.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools.Test/LookupPipelineTests.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools.Test/MapXmlLookupToGetByExampleJsonStepTests.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupPipeline/XmlToWorkItemsProcessorTests.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupToResourceProcessorTests.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/EducationOrganizationIdentityCache.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/Factories/XmlIoFactory.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/IEducationOrganizationIdentityCache.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/AvoidDuplicateLookupsStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/ComputeHashStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/DirectLookupToIdentityMappingStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/EducationOrganizationCacheLookupStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/IdentifyResourceTypeStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapResourceToIdentityStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapXmlLookupToGetByExampleJsonStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/PerformGetByExampleStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/StoreIdentityForWritingStep.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupPipelineProcessor.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupToResourceProcessor.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupWorkItem.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlToWorkItemsProcessor.cs delete mode 100644 Utilities/DataLoading/EdFi.LoadTools/XmlLookupApplication.cs diff --git a/Utilities/DataLoading/EdFi.LoadTools.Test/EducationOrganizationCacheLookupStepTests.cs b/Utilities/DataLoading/EdFi.LoadTools.Test/EducationOrganizationCacheLookupStepTests.cs deleted file mode 100644 index 10ae55515f..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools.Test/EducationOrganizationCacheLookupStepTests.cs +++ /dev/null @@ -1,166 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.Xml.Linq; -using EdFi.LoadTools.Engine; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using NUnit.Framework; - -namespace EdFi.LoadTools.Test -{ - [TestFixture] - public class EducationOrganizationCacheLookupStepTests - { - private static Dictionary> _lookupPropertyValuesByName; - - private readonly string _educationOrganizationReferenceElement = @" - - 255950 - uri://ed-fi.org/EducationOrganizationIdentificationSystemDescriptor#NCES - - Region 9 - uri://ed-fi.org/EducationOrganizationCategoryDescriptor#School - "; - - [Test] - public void Should_correctly_build_lookup_property_dictionary_from_LookupXml() - { - var xelement = XElement.Parse(_educationOrganizationReferenceElement); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "EducationOrganization", LookupName = "EducationOrganizationLookup", - IdentityName = "EducationOrganizationIdentity" - }; - - var step = new EducationOrganizationCacheLookupStep( - new TestEducationOrganizationIdentityCache( - new List - { - 9999 - })); - - step.Process(item); - - Assert.AreEqual(_lookupPropertyValuesByName.Count, 4); - - foreach (var expectedKey in new List - { - "EducationOrganizationCategory", "EducationOrganizationIdentificationSystem", "IdentificationCode", - "NameOfInstitution" - }) - { - Assert.IsTrue(_lookupPropertyValuesByName.ContainsKey(expectedKey)); - } - } - - [Test] - public void Should_correctly_set_IdentityXElement_if_EdOrg_found_in_identity_cache() - { - var xelement = XElement.Parse(_educationOrganizationReferenceElement); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "EducationOrganization", LookupName = "EducationOrganizationLookup", - IdentityName = "EducationOrganizationIdentity" - }; - - var step = new EducationOrganizationCacheLookupStep( - new TestEducationOrganizationIdentityCache( - new List - { - 9999 - })); - - step.Process(item); - - Assert.AreEqual( - item.IdentityXElement.ToString(), - new XElement(item.IdentityName, new XElement($"{item.ResourceName}Id", 9999)).ToString()); - } - - [Test] - public void Should_not_set_IdentityXElement_if_no_EdOrg_is_found_in_cache() - { - var xelement = XElement.Parse(_educationOrganizationReferenceElement); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "EducationOrganization", LookupName = "EducationOrganizationLookup", - IdentityName = "EducationOrganizationIdentity" - }; - - var step = new EducationOrganizationCacheLookupStep( - new TestEducationOrganizationIdentityCache(new List())); - - step.Process(item); - - Assert.IsNull(item.IdentityXElement); - } - - [Test] - public void Should_not_set_IdentityXElement_more_than_1_edorg_is_found_in_cache_for_provided_lookup_columns() - { - var xelement = XElement.Parse(_educationOrganizationReferenceElement); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "EducationOrganization", LookupName = "EducationOrganizationLookup", - IdentityName = "EducationOrganizationIdentity" - }; - - var step = new EducationOrganizationCacheLookupStep( - new TestEducationOrganizationIdentityCache( - new List - { - 123, 456 - })); - - step.Process(item); - - Assert.IsNull(item.IdentityXElement); - } - - [Test] - public void Should_only_process_education_organization_lookups() - { - var xelement = XElement.Parse(_educationOrganizationReferenceElement); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "EducationOrganization", LookupName = "NotEducationOrganizationLookup", - IdentityName = "EducationOrganizationIdentity" - }; - - var step = new EducationOrganizationCacheLookupStep( - new TestEducationOrganizationIdentityCache( - new List - { - 123, 456 - })); - - step.Process(item); - - Assert.IsNull(item.IdentityXElement); - } - - private class TestEducationOrganizationIdentityCache : IEducationOrganizationIdentityCache - { - private readonly List _educationOrganizationIds; - - public TestEducationOrganizationIdentityCache(List educationOrganizationIds) - { - _educationOrganizationIds = educationOrganizationIds; - } - - public IEnumerable Get(Dictionary> lookupPropertyValuesByName) - { - _lookupPropertyValuesByName = lookupPropertyValuesByName; - return _educationOrganizationIds; - } - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools.Test/LookupPipelineTests.cs b/Utilities/DataLoading/EdFi.LoadTools.Test/LookupPipelineTests.cs deleted file mode 100644 index 0b37a4ecff..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools.Test/LookupPipelineTests.cs +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Xml; -using System.Xml.Linq; -using EdFi.LoadTools.Engine; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using NUnit.Framework; - -namespace EdFi.LoadTools.Test -{ - [TestFixture] - public class LookupPipelineTests - { - public IEnumerable Pipeline(IDictionary hashIdentities) - { - var hashProvider = new HashProvider(); - hashIdentities = hashIdentities ?? new ConcurrentDictionary(); - - return new ILookupPipelineStep[] - { - new IdentifyResourceTypeStep(), new ComputeHashStep(hashProvider), new AvoidDuplicateLookupsStep(hashIdentities), - - //new MapXmlLookupToGetByExampleJsonStep(), - //new PerformGetByExampleStep(), - //new MapResourceToIdentityStep(), - new StoreIdentityForWritingStep(hashIdentities) - }; - } - - [Test] - public void Should_not_perform_lookup_multiple_times_for_the_same_value() - { - var xml = XElement.Parse( - "" + - "A" + - "" + - "A" + - "" + - ""); - - var xwip = new XmlToWorkItemsProcessor(); - var items = xwip.GetLookupWorkItems(new XmlTextReader(new StringReader(xml.ToString()))).ToList(); - var hashIdentities = new Dictionary(); - - foreach (var xmlLookupWorkItem in items) - { - foreach (var lookupPipelineStep in Pipeline(hashIdentities)) - { - lookupPipelineStep.Process(xmlLookupWorkItem); - } - } - - foreach (var wi in items) - { - Console.WriteLine(wi.HashString); - Console.WriteLine(wi.LookupXElement); - Console.WriteLine(wi.IdentityXElement); - } - - Assert.AreEqual(3, items.Count); - Assert.AreEqual(2, hashIdentities.Count); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools.Test/MapXmlLookupToGetByExampleJsonStepTests.cs b/Utilities/DataLoading/EdFi.LoadTools.Test/MapXmlLookupToGetByExampleJsonStepTests.cs deleted file mode 100644 index 3e7f48c076..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools.Test/MapXmlLookupToGetByExampleJsonStepTests.cs +++ /dev/null @@ -1,210 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Xml.Linq; -using EdFi.LoadTools.Engine.Mapping; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using EdFi.LoadTools.Test.MappingFactories; -using NUnit.Framework; - -namespace EdFi.LoadTools.Test -{ - [TestFixture] - public class MapXmlLookupToGetByExampleJsonStepTests - { - private MapXmlLookupToGetByExampleJsonStep _mapStep; - - [OneTimeSetUp] - public void Setup() - { - var mappings = new[] - { - new MetadataMapping - { - RootName = "Simple", SourceName = "Abc", TargetName = "alphabet", Properties = new List - { - new PropertyMapping - { - IsArray = false, - SourceName = "A", - SourceType = "String", - TargetName = "a", - TargetType = "string", - MappingStrategy = - new - CopySimplePropertyMappingStrategy() - }, - new PropertyMapping - { - IsArray = false, - SourceName = "B", - SourceType = "String", - TargetName = "b", - TargetType = "string", - MappingStrategy = - new - CopySimplePropertyMappingStrategy() - }, - new PropertyMapping - { - IsArray = false, - SourceName = "C", - SourceType = "String", - TargetName = "c", - TargetType = "string", - MappingStrategy = - new - CopySimplePropertyMappingStrategy() - } - } - }, - new MetadataMapping - { - RootName = "Descriptor", SourceName = "Foo", TargetName = "foo", Properties = new List - { - new PropertyMapping - { - IsArray = false, - SourceName = "FooDescriptor", - SourceType = - "FooDescriptorReferenceType", - TargetName = "fooDescriptor", - TargetType = "string", - MappingStrategy = - new - DescriptorReferenceTypeToStringMappingStrategy() - }, - new PropertyMapping - { - IsArray = false, SourceName = - "FooDescriptor/Namespace", - SourceType = "String", - TargetName = "{none}", - TargetType = "{none}", - MappingStrategy = - new - NoOperationMappingStrategy() - }, - new PropertyMapping - { - IsArray = false, SourceName = - "FooDescriptor/CodeValue", - SourceType = "String", - TargetName = "{none}", - TargetType = "{none}", - MappingStrategy = - new - NoOperationMappingStrategy() - } - } - }, - new MetadataMapping - { - RootName = "Array", SourceName = "MyArray", TargetName = "myArray", Properties = new List - { - new PropertyMapping - { - IsArray = true, - SourceName = "A", - SourceType = "String", - TargetName = "a", - TargetType = "string", - MappingStrategy = - new - ArrayToArrayMappingStrategy() - }, - new PropertyMapping - { - IsArray = true, - SourceName = "A/B", - SourceType = "String", - TargetName = "a/b", - TargetType = "string", - MappingStrategy = - new - CopySimplePropertyMappingStrategy() - } - } - } - }; - - var mappingFactory = new MockMetadataMappingFactory(mappings); - - _mapStep = new MapXmlLookupToGetByExampleJsonStep(mappingFactory); - } - - [Test] - public void Should_perform_simple_element_name_translations() - { - var xelement = new XElement( - "Abc", new XElement("A", "hello"), new XElement("B", "world"), - new XElement("C", "!!!")); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "Simple" - }; - - Console.WriteLine(item.LookupXElement); - var success = _mapStep.Process(item).Result; - Assert.IsTrue(success); - Console.WriteLine(item.GetByExampleXElement); - - var expected = new XElement( - "alphabet", new XElement("a", "hello"), new XElement("b", "world"), - new XElement("c", "!!!")); - - Assert.AreEqual(expected.ToString(), item.GetByExampleXElement.ToString()); - } - - [Test] - public void Should_perform_reference_type_translations() - { - var xelement = new XElement("Foo", new XElement("FooDescriptor", "NameSpace#descriptor")); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "Descriptor" - }; - - Console.WriteLine(item.LookupXElement); - var success = _mapStep.Process(item).Result; - Assert.IsTrue(success); - Console.WriteLine(item.GetByExampleXElement); - var expected = new XElement("foo", new XElement("fooDescriptor", "NameSpace#descriptor")); - Assert.AreEqual(expected.ToString(), item.GetByExampleXElement.ToString()); - } - - [Test] - public void Should_perform_array_translations() - { - XNamespace json = "http://james.newtonking.com/projects/json"; - - var xelement = new XElement( - "MyArray", - new XElement("A", new XElement("B", "1"), new XElement("B", "2"), new XElement("B", "3"))); - - var item = new XmlLookupWorkItem(xelement) - { - ResourceName = "Array" - }; - - Console.WriteLine(item.LookupXElement); - var success = _mapStep.Process(item).Result; - Assert.IsTrue(success); - Console.WriteLine(item.GetByExampleXElement); - - var expected = new XElement( - "myArray", - new XElement( - "a", new XAttribute(json + "Array", true), new XElement("b", "1"), new XElement("b", "2"), - new XElement("b", "3"))); - - Assert.AreEqual(expected.ToString(), item.GetByExampleXElement.ToString()); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupPipeline/XmlToWorkItemsProcessorTests.cs b/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupPipeline/XmlToWorkItemsProcessorTests.cs deleted file mode 100644 index daec57a08e..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupPipeline/XmlToWorkItemsProcessorTests.cs +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.IO; -using System.Linq; -using System.Text; -using System.Xml; -using System.Xml.Linq; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using NUnit.Framework; - -namespace EdFi.LoadTools.Test.XmlLookupPipeline -{ - [TestFixture] - public class XmlToWorkItemsProcessorTests - { - private readonly XElement _lookupInsideIdentity = XElement.Parse( - @" - 2015-11-07 - - 640 - Integer - College Board examination scores - - - - 605750 - - - Some Lookup Name - - - - - 605750 - - Kent - Wyatt - - - 1995-10-31 - - - - 255901001 - - - - - - - SAT-HighestCompositeSection - - Writing - - - Twelfth grade - - - - -"); - - private readonly XElement _nestedLookups = XElement.Parse( - @" - 2015-11-07 - - 640 - Integer - College Board examination scores - - - - 605750 - - Kent - Wyatt - - - 1995-10-31 - - - - 255901001 - - - - - - - SAT-HighestCompositeSection - - Writing - - - Twelfth grade - - - - -"); - - private readonly XElement _resolvedLookup = XElement.Parse( - @" - 2015-11-07 - - 640 - Integer - College Board examination scores - - - - 605750 - - - 605750 - - Kent - Wyatt - - - 1995-10-31 - - - - 255901001 - - - - - - - SAT-HighestCompositeSection - - Writing - - - Twelfth grade - - - - -"); - - private XmlLookupWorkItem[] TestWorkItems(XElement xElement) - { - var processor = new XmlToWorkItemsProcessor(); - var stream = new MemoryStream(Encoding.UTF8.GetBytes(xElement.ToString())); - var reader = new XmlTextReader(stream); - var workItems = processor.GetLookupWorkItems(reader).ToArray(); - - foreach (var xmlLookupWorkItem in workItems) - { - Console.WriteLine(xmlLookupWorkItem.LookupXElement); - } - - return workItems; - } - - [Test] - public void Should_have_one_work_items_lookup() - { - var items = TestWorkItems(_resolvedLookup); - Assert.AreEqual(1, items.Length); - } - - [Test] - public void Should_have_two_work_items_lookup() - { - var items = TestWorkItems(_lookupInsideIdentity); - Assert.AreEqual(2, items.Length); - } - - [Test] - public void Should_have_three_work_items_lookups() - { - var items = TestWorkItems(_nestedLookups); - Assert.AreEqual(3, items.Length); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupToResourceProcessorTests.cs b/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupToResourceProcessorTests.cs deleted file mode 100644 index c9fd0ebc7c..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools.Test/XmlLookupToResourceProcessorTests.cs +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Xml; -using System.Xml.Linq; -using EdFi.LoadTools.Engine; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using NUnit.Framework; - -namespace EdFi.LoadTools.Test -{ - [TestFixture] - public class XmlLookupToResourceProcessorTests - { - private readonly XElement _a1Identity = XElement.Parse("1"); - - private readonly XElement _a1Lookup = XElement.Parse("A"); - private readonly XElement _a2Identity = XElement.Parse("2"); - private readonly XElement _a2Lookup = XElement.Parse("B"); - private readonly XElement _bLookup = XElement.Parse("B"); - private readonly IHashProvider _hashProvider = new HashProvider(); - - private readonly Func _x2Hs = (h, x) => h.BytesToStr(h.Hash(x.ToString())); - - private IDictionary HashIdentities - { - get - { - var result = new Dictionary - { - { - _x2Hs(_hashProvider, _a1Lookup), _a1Identity - }, - { - _x2Hs(_hashProvider, _a2Lookup), _a2Identity - } - }; - - return result; - } - } - - private XElement RunProcessorOnInputElement(XElement source) - { - var processor = new XmlLookupToResourceProcessor( - _hashProvider, - HashIdentities); - - var reader = new XmlTextReader(new StringReader(source.ToString())); - var sb = new StringBuilder(); - var writer = new XmlTextWriter(new StringWriter(sb)); - - processor.CopyXmlLookupsToResources(reader, writer); - - var xmlResult = XElement.Parse(sb.ToString()); - return xmlResult; - } - - [Test] - public void Should_add_identity_for_successful_lookup() - { - var xmlSource = XElement.Parse("" + _a1Lookup + ""); - var xmlResult = RunProcessorOnInputElement(xmlSource); - Console.WriteLine($"xmlSource:\r\n {xmlSource}\r\n\r\nxmlResult:\r\n {xmlResult}"); - Assert.IsNotNull(xmlResult.Element("AIdentity")); - } - - [Test] - public void Should_ignore_identity_for_no_lookup() - { - var xmlSource = XElement.Parse("" + _a1Identity + ""); - var xmlResult = RunProcessorOnInputElement(xmlSource); - Console.WriteLine($"xmlSource:\r\n {xmlSource}\r\n\r\nxmlResult:\r\n {xmlResult}"); - Assert.IsFalse(xmlResult.Nodes().Any(x => x.NodeType == XmlNodeType.Comment)); - } - - [Test] - public void Should_prefer_provided_identity_information_over_lookup() - { - var xmlSource = XElement.Parse("" + _a1Identity + _a1Lookup + ""); - var xmlResult = RunProcessorOnInputElement(xmlSource); - Console.WriteLine($"xmlSource:\r\n {xmlSource}\r\n\r\nxmlResult:\r\n {xmlResult}"); - var identities = xmlResult.Descendants("AIdentity").ToArray(); - Assert.IsTrue(identities.All(x => x.PreviousNode == null)); - } - - [Test] - public void Should_show_failure_comment_for_unsuccessful_lookup() - { - var xmlSource = XElement.Parse("" + _bLookup + ""); - var xmlResult = RunProcessorOnInputElement(xmlSource); - Console.WriteLine($"xmlSource:\r\n {xmlSource}\r\n\r\nxmlResult:\r\n {xmlResult}"); - Assert.IsNull(xmlResult.Element("BIdentity")); - Assert.IsTrue(xmlResult.Nodes().Any(x => x.NodeType == XmlNodeType.Comment)); - Assert.IsTrue(xmlResult.ToString().Contains("No BIdentity could be retrieved")); - } - - [Test] - public void Should_recurse_identity_lookups() - { - var xmlSource = XElement.Parse( - "" + - "" + _a1Lookup + "" + - "" + _a2Lookup + "" + - "" + _a1Lookup + "" + - ""); - - var xmlResult = RunProcessorOnInputElement(xmlSource); - Console.WriteLine($"xmlSource:\r\n {xmlSource}\r\n\r\nxmlResult:\r\n {xmlResult}"); - var identities = xmlResult.Descendants("AIdentity").ToArray(); - Assert.AreEqual(3, identities.Length); - Assert.IsTrue(identities.All(x => x.PreviousNode.NodeType == XmlNodeType.Comment)); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/EducationOrganizationIdentityCache.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/EducationOrganizationIdentityCache.cs deleted file mode 100644 index 7dc6c24cce..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/EducationOrganizationIdentityCache.cs +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; -using System.Xml.Schema; -using EdFi.Common.Extensions; -using EdFi.LoadTools.ApiClient; -using EdFi.LoadTools.Engine.XmlLookupPipeline; -using Newtonsoft.Json.Linq; - -namespace EdFi.LoadTools.Engine -{ - public class EducationOrganizationIdentityCache : IEducationOrganizationIdentityCache - { - private readonly IOdsRestClient _client; - private readonly Lazy> _educationOrganizations; - private readonly XmlSchemaSet _schemaSet; - - public EducationOrganizationIdentityCache(XmlSchemaSet schemaSet, IOdsRestClient client) - { - _schemaSet = schemaSet; - _client = client; - _educationOrganizations = new Lazy>(Initialize); - } - - public IEnumerable Get(Dictionary> lookupPropertyValuesByName) - { - // Foreach lookup property entry, find all EdOrgs in cache which satisfy each predicate. - return _educationOrganizations.Value.Where( - e => GetPropertyPredicates(lookupPropertyValuesByName).All(p => p(e))) - .Select(o => (int) o["educationOrganizationId"]); - } - - private List Initialize() - { - var educationOrganizations = new List(); - - foreach (XmlSchemaType element in _schemaSet.GlobalTypes.Values) - - // For all types derived from EducationOrganization - { - if (element.BaseXmlSchemaType?.Name?.Equals("EducationOrganization") ?? false) - { - var offset = 0; - - while (true) - { - // Until no results are returned, perform GetAll on type and cache response payloads. - var jarrayTask = Task.Run(() => GetElementJArray(element.Name, offset)); - jarrayTask.Wait(); - - var jArray = jarrayTask.Result; - - foreach (var jObject in jArray.Children()) - { - // Replace EducationOrganizationId with correct Id for type. SchoolId, LocalEducationAgencyId etc. - var educationOrganizationId = - (string) jObject[$"{element.Name}Id".ToCamelCase()]; - - jObject.Add(new JProperty("educationOrganizationId", educationOrganizationId)); - jObject.Remove($"{element.Name}Id".ToCamelCase()); - educationOrganizations.Add(jObject); - } - - if (jArray.Any()) - { - offset += jArray.Count; - } - else - { - break; - } - } - } - } - - return educationOrganizations; - } - - private async Task GetElementJArray(string elementName, int offset) - { - var response = _client.GetAll(elementName, offset).Result; - var json = await response.Content.ReadAsStringAsync(); - return JArray.Parse(json); - } - - public IEnumerable ResolveLookupProperty(string lookupPropertyName, JToken jToken) - { - // Recursively traverse EdOrg json and discover all property values by name. - if (jToken.Type == JTokenType.Object) - { - foreach (var child in jToken.Children()) - { - if (child.Name.Replace("Descriptor", string.Empty) - .Equals(lookupPropertyName, StringComparison.CurrentCultureIgnoreCase)) - { - yield return child.Value.Value(); - } - - foreach (var property in ResolveLookupProperty(lookupPropertyName, child.Value)) - { - yield return property; - } - } - } - else if (jToken.Type == JTokenType.Array) - { - foreach (var child in jToken.Children()) - { - foreach (var property in ResolveLookupProperty(lookupPropertyName, child)) - { - yield return property; - } - } - } - } - - public IList> GetPropertyPredicates( - Dictionary> lookupPropertyValuesByName) - { - var conditions = new List>(); - - foreach (var lookupProperty in lookupPropertyValuesByName) - - // For each EdOrg, resolve lookup property by name. - // Ensure EdOrg values match lookupProperty values. - { - conditions.Add( - o => ResolveLookupProperty(lookupProperty.Key, o) - .OrderBy(x => x) - .SequenceEqual(lookupProperty.Value.Select(lp => lp.Value).OrderBy(x => x))); - } - - return conditions; - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/Factories/XmlIoFactory.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/Factories/XmlIoFactory.cs deleted file mode 100644 index 8ce9ff4754..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/Factories/XmlIoFactory.cs +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Xml; - -namespace EdFi.LoadTools.Engine.Factories -{ - public class XmlIoFactory : IXmlPairsFactory - { - private readonly string _destinationPath; - private readonly List _files; - - public XmlIoFactory(IDataConfiguration configuration) - { - _destinationPath = Path.Combine(configuration.Folder, "Output"); - _files = new List(Directory.GetFiles(configuration.Folder, "*.xml")); - - var dirInfo = Directory.CreateDirectory(_destinationPath); - - foreach (var fileInfo in dirInfo.EnumerateFiles()) - { - if (_files.Any( - f => - string.Compare(Path.GetFileName(f), fileInfo.Name, StringComparison.CurrentCultureIgnoreCase) == 0)) - { - fileInfo.Delete(); - } - } - } - - public IEnumerable GetSources() - { - var result = _files.Select(f => new XmlTextReader(f)); - return result; - } - - public IEnumerable GetIoPairs() - { - var result = _files.Select( - f => new XmlIoPair - { - Source = new XmlTextReader(f), - - // ReSharper disable once AssignNullToNotNullAttribute - Destination = new XmlTextWriter(Path.Combine(_destinationPath, Path.GetFileName(f)), Encoding.UTF8) - }); - - return result; - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/IEducationOrganizationIdentityCache.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/IEducationOrganizationIdentityCache.cs deleted file mode 100644 index 524ecc907a..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/IEducationOrganizationIdentityCache.cs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using EdFi.LoadTools.Engine.XmlLookupPipeline; - -namespace EdFi.LoadTools.Engine -{ - public interface IEducationOrganizationIdentityCache - { - IEnumerable Get(Dictionary> lookupPropertyValuesByName); - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/Interfaces.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/Interfaces.cs index bb47f61bd5..ff29b4bd9a 100644 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/Interfaces.cs +++ b/Utilities/DataLoading/EdFi.LoadTools/Engine/Interfaces.cs @@ -7,12 +7,9 @@ using System.IO; using System.Net; using System.Reflection; -using System.Threading.Tasks; -using System.Xml; using System.Xml.Linq; using EdFi.LoadTools.ApiClient; using EdFi.LoadTools.Engine.Mapping; -using EdFi.LoadTools.Engine.XmlLookupPipeline; namespace EdFi.LoadTools.Engine { @@ -206,18 +203,6 @@ public interface IXmlReferenceCache XElement VisitReference(string id); } - public interface ILookupPipelineStep - { - Task Process(XmlLookupWorkItem item); - } - - public interface IXmlPairsFactory - { - IEnumerable GetSources(); - - IEnumerable GetIoPairs(); - } - public interface IXsdConfiguration { string Folder { get; } diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/AvoidDuplicateLookupsStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/AvoidDuplicateLookupsStep.cs deleted file mode 100644 index 3385f9ec00..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/AvoidDuplicateLookupsStep.cs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.Threading.Tasks; -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class AvoidDuplicateLookupsStep : ILookupPipelineStep - { - private readonly IDictionary _hashIdentities; - - public AvoidDuplicateLookupsStep(IDictionary hashIdentities) - { - _hashIdentities = hashIdentities; - } - -#pragma warning disable 1998 - public async Task Process(XmlLookupWorkItem item) -#pragma warning restore 1998 - { - if (_hashIdentities.ContainsKey(item.HashString)) - { - return false; - } - - _hashIdentities.Add(item.HashString, item.IdentityXElement); - return true; - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/ComputeHashStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/ComputeHashStep.cs deleted file mode 100644 index 3557739fe0..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/ComputeHashStep.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Threading.Tasks; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class ComputeHashStep : ILookupPipelineStep - { - private readonly IHashProvider _hashProvider; - - public ComputeHashStep(IHashProvider hashProvider) - { - _hashProvider = hashProvider; - } - - public Task Process(XmlLookupWorkItem item) - { - var hashBytes = _hashProvider.Hash(item.LookupXElement.ToString()); - var hash = _hashProvider.BytesToStr(hashBytes); - item.HashString = hash; - return Task.FromResult(true); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/DirectLookupToIdentityMappingStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/DirectLookupToIdentityMappingStep.cs deleted file mode 100644 index dbfd584b51..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/DirectLookupToIdentityMappingStep.cs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Linq; -using System.Threading.Tasks; -using System.Xml.Linq; -using EdFi.LoadTools.Engine.Mapping; -using log4net; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - /// - /// If they've provided the required Identity values in the Lookup, just map them directly - /// - public class DirectLookupToIdentityMappingStep : ILookupPipelineStep - { - private readonly MetadataMapping[] _mappings; - - // ReSharper disable once InconsistentNaming - private readonly ILog Log; - - public DirectLookupToIdentityMappingStep(IMetadataMappingFactory metadataMappingFactory) - { - _mappings = metadataMappingFactory.GetMetadataMappings().ToArray(); - Log = LogManager.GetLogger(GetType().Name); - } - - public Task Process(XmlLookupWorkItem item) - { - if (item.IdentityXElement != null) - { - return Task.FromResult(true); - } - - var map = _mappings.SingleOrDefault(m => m.RootName == item.ResourceName); - - if (map == null) - { - Log.Warn($"No mappings for {item.ResourceName} "); - return Task.FromResult(false); - } - - if (LookupHasAllValuesInMap(item.LookupXElement, map)) - { - var identityXElement = new XElement(item.IdentityName); - - foreach (var element in item.LookupXElement.Elements()) - { - var mapping = map.Properties.SingleOrDefault(x => x.SourceName == element.Name.LocalName); - mapping?.MappingStrategy.MapElement(element, mapping.SourceName, identityXElement); - } - - item.IdentityXElement = identityXElement; - } - - return Task.FromResult(true); - } - - private static bool LookupHasAllValuesInMap(XElement lookupXElement, MetadataMapping map) - { - var ns = lookupXElement.Name.Namespace; - - return map.Properties - .Select(propertyMapping => lookupXElement.Element(ns + propertyMapping.SourceName)) - .All(xElement => xElement != null && !xElement.IsEmpty); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/EducationOrganizationCacheLookupStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/EducationOrganizationCacheLookupStep.cs deleted file mode 100644 index da368e90fc..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/EducationOrganizationCacheLookupStep.cs +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class EducationOrganizationCacheLookupStep : ILookupPipelineStep - { - private readonly IEducationOrganizationIdentityCache _educationOrganizationIdentityCache; - - public EducationOrganizationCacheLookupStep( - IEducationOrganizationIdentityCache educationOrganizationIdentityCache) - { - _educationOrganizationIdentityCache = educationOrganizationIdentityCache; - } - - public Task Process(XmlLookupWorkItem item) - { - if (item.LookupXElement == null) - { - return Task.FromResult(true); - } - - // Only process EdOrg Lookups - if (!item.LookupName?.Equals("EducationOrganizationLookup") ?? true) - { - return Task.FromResult(true); - } - - // Recursively build list of lookup properties for EdOrg - var lookupProperties = GetLookupProperties(item.LookupXElement); - - // Search cache using lookup properties, grouping by name in the case of collections. - var result = _educationOrganizationIdentityCache.Get( - lookupProperties - .GroupBy(p => p.Name) - .ToDictionary( - k => k.Key, - v => v.ToList())) - .ToList(); - - // If 0 or more than one EdOrg matches lookup, return. - if (!result.Any() || result.Count() > 1) - { - return Task.FromResult(true); - } - - item.IdentityXElement = - new XElement(item.IdentityName, new XElement($"{item.ResourceName}Id", result.First())); - - return Task.FromResult(true); - } - - private List GetLookupProperties(XElement lookupElement) - { - return lookupElement.Elements() - .Where(e => !e.HasElements) - .Select( - e => new LookupProperty - { - Name = e.Name.LocalName, Value = e.Value - }) - .Concat( - lookupElement.Elements() - .Where(e => e.HasElements) - .SelectMany(GetLookupProperties)) - .ToList(); - } - } - - public class LookupProperty - { - public string Name { get; set; } - - public string Value { get; set; } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/IdentifyResourceTypeStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/IdentifyResourceTypeStep.cs deleted file mode 100644 index 6d151a4310..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/IdentifyResourceTypeStep.cs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using EdFi.LoadTools.Common; -using EdFi.Common.Inflection; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class IdentifyResourceTypeStep : ILookupPipelineStep - { - private readonly Regex _lookupRegex = new Regex(Constants.LookupRegex); - - public Task Process(XmlLookupWorkItem item) - { - var match = _lookupRegex.Match(item.LookupXElement.Name.LocalName); - - if (!match.Success) - { - return Task.FromResult(false); - } - - item.ResourceName = match.Groups["TypeName"].Value; - item.IdentityName = $"{item.ResourceName}Identity"; - item.LookupName = $"{item.ResourceName}Lookup"; - item.JsonResourceName = Inflector.MakePlural(Inflector.MakeInitialLowerCase(item.ResourceName)); - item.JsonModelName = Inflector.MakeInitialLowerCase(item.ResourceName); - return Task.FromResult(true); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapResourceToIdentityStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapResourceToIdentityStep.cs deleted file mode 100644 index 06a0c8a13e..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapResourceToIdentityStep.cs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Linq; -using System.Threading.Tasks; -using System.Xml.Linq; -using EdFi.LoadTools.Engine.Mapping; -using log4net; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class MapResourceToIdentityStep : ILookupPipelineStep - { - private readonly MetadataMapping[] _mappings; - - // ReSharper disable once InconsistentNaming - private readonly ILog Log; - - public MapResourceToIdentityStep(IMetadataMappingFactory metadataMappingFactory) - { - _mappings = metadataMappingFactory.GetMetadataMappings().ToArray(); - Log = LogManager.GetLogger(GetType().Name); - } - - public Task Process(XmlLookupWorkItem item) - { - if (item.IdentityXElement != null) - { - return Task.FromResult(true); - } - - var map = _mappings.SingleOrDefault(m => m.RootName == item.ResourceName); - - if (map == null) - { - Log.Warn($"No mappings for {item.ResourceName} "); - return Task.FromResult(false); - } - - var targetXElement = new XElement(item.IdentityName); - - foreach (var element in item.ResourceXElement.Elements()) - { - var path = element.Name.LocalName; - PerformElementReverseMapping(map, element, path, targetXElement); - } - - item.IdentityXElement = targetXElement; - return Task.FromResult(true); - } - - private void PerformElementReverseMapping(MetadataMapping map, XElement sourceElement, string path, - XElement targetElement) - { - var propertyMappings = map.Properties.Where(p => p.TargetName == path); - - foreach (var mapping in propertyMappings) - { - mapping.MappingStrategy.ReverseMapElement(sourceElement, mapping.SourceName, targetElement); - } - - foreach (var ele in sourceElement.Elements()) - { - PerformElementReverseMapping(map, ele, $"{path}/{ele.Name.LocalName}", targetElement); - } - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapXmlLookupToGetByExampleJsonStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapXmlLookupToGetByExampleJsonStep.cs deleted file mode 100644 index 47bf9917bf..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/MapXmlLookupToGetByExampleJsonStep.cs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Linq; -using System.Threading.Tasks; -using System.Xml.Linq; -using EdFi.LoadTools.Engine.Mapping; -using log4net; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class MapXmlLookupToGetByExampleJsonStep : ILookupPipelineStep - { - private readonly MetadataMapping[] _mappings; - - // ReSharper disable once InconsistentNaming - private readonly ILog Log; - - public MapXmlLookupToGetByExampleJsonStep(IMetadataMappingFactory metadataMappingFactory) - { - _mappings = metadataMappingFactory.GetMetadataMappings().ToArray(); - Log = LogManager.GetLogger(GetType().Name); - } - - public Task Process(XmlLookupWorkItem item) - { - if (item.IdentityXElement != null) - { - return Task.FromResult(true); - } - - var map = _mappings.SingleOrDefault(m => m.RootName == item.ResourceName); - - if (map == null) - { - Log.Warn($"No mappings for {item.ResourceName} "); - return Task.FromResult(false); - } - - var lookupXElement = new XElement(map.TargetName); - - foreach (var element in item.LookupXElement.Elements()) - { - var path = element.Name.LocalName; - PerformElementMapping(map, element, path, lookupXElement); - } - - item.GetByExampleXElement = lookupXElement; - return Task.FromResult(true); - } - - private void PerformElementMapping(MetadataMapping map, XElement sourceElement, string path, - XElement targetElement) - { - var propertyMappings = map.Properties.Where(p => p.SourceName == path); - - foreach (var mapping in propertyMappings) - { - mapping.MappingStrategy.MapElement(sourceElement, mapping.TargetName, targetElement); - } - - foreach (var ele in sourceElement.Elements()) - { - PerformElementMapping(map, ele, $"{path}/{ele.Name.LocalName}", targetElement); - } - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/PerformGetByExampleStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/PerformGetByExampleStep.cs deleted file mode 100644 index 2342688fe1..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/PerformGetByExampleStep.cs +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System; -using System.Threading.Tasks; -using EdFi.LoadTools.ApiClient; -using log4net; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - /// - /// resolves the item's GetByExampleXElement to it's ResourceXElement - /// - public class PerformGetByExampleStep : ILookupPipelineStep - { - private readonly IOdsRestClient _client; - - // ReSharper disable once InconsistentNaming - private readonly ILog Log; - - public PerformGetByExampleStep(IOdsRestClient client) - { - Log = LogManager.GetLogger(GetType().Name); - _client = client; - } - - public async Task Process(XmlLookupWorkItem item) - { - if (item.IdentityXElement != null) - { - return true; - } - - var jsonText = JsonConvert.SerializeXNode(item.GetByExampleXElement); - var response = await _client.GetResourceByExample(jsonText, item.JsonModelName).ConfigureAwait(false); - var jsonLookupResult = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - if (response.IsSuccessStatusCode) - { - try - { - var token = JToken.Parse(jsonLookupResult); - - if (token is JObject) - { - item.ResourceXElement = JsonConvert.DeserializeXNode(jsonLookupResult, item.JsonModelName).Root; - } - else if (token is JArray) - { - var jArr = token as JArray; - - if (jArr.Count == 0) - { - Log.Error($"{item.ResourceName} lookup returned no matching resources"); - Log.Debug(response.RequestMessage); - return false; - } - - if (jArr.Count > 1) - { - Log.Error($"{item.ResourceName} lookup returned more than one matching resources"); - Log.Debug(response.RequestMessage); - return false; - } - - item.ResourceXElement = JsonConvert.DeserializeXNode( - jArr.First.ToString(), - item.JsonModelName).Root; - } - - return true; - } - catch (Exception e) - { - Log.Fatal(jsonLookupResult, e); - throw; - } - } - - Log.Error($"{item.ResourceName} lookup error: {response.ReasonPhrase}"); - Log.Info(jsonLookupResult); - Log.Debug(response.RequestMessage); - return false; - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/StoreIdentityForWritingStep.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/StoreIdentityForWritingStep.cs deleted file mode 100644 index ff9d867584..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/StoreIdentityForWritingStep.cs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.Threading.Tasks; -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - /// - /// When we hit this step, we have an identity to replace the lookup; save it for later. - /// - public class StoreIdentityForWritingStep : ILookupPipelineStep - { - private readonly IDictionary _hashIdentities; - - public StoreIdentityForWritingStep(IDictionary hashIdentities) - { - _hashIdentities = hashIdentities; - } - - public Task Process(XmlLookupWorkItem item) - { - _hashIdentities[item.HashString] = item.IdentityXElement; - return Task.FromResult(true); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupPipelineProcessor.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupPipelineProcessor.cs deleted file mode 100644 index 41ff5dd3a2..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupPipelineProcessor.cs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Linq; -using System.Threading.Tasks.Dataflow; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class XmlLookupPipelineProcessor - { - private readonly ILookupPipelineStep[] _lookupPipelineSteps; - - public XmlLookupPipelineProcessor(ILookupPipelineStep[] lookupPipelineSteps) - { - _lookupPipelineSteps = lookupPipelineSteps; - } - - public ITargetBlock CreateBlock() - { - return new ActionBlock(x => RunPipelineForWorkItem(x)); - } - - public bool RunPipelineForWorkItem(XmlLookupWorkItem item) - { - return _lookupPipelineSteps.All(s => s.Process(item).Result); - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupToResourceProcessor.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupToResourceProcessor.cs deleted file mode 100644 index 130174e0f0..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupToResourceProcessor.cs +++ /dev/null @@ -1,202 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.IO; -using System.Text.RegularExpressions; -using System.Threading.Tasks.Dataflow; -using System.Xml; -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class XmlLookupToResourceProcessor - { - private readonly IDictionary _hashIdentities; - private readonly IHashProvider _hashProvider; - - private readonly Regex _regex = new Regex(Constants.ReferenceRegex); - - public XmlLookupToResourceProcessor(IHashProvider hashProvider, IDictionary hashIdentities) - { - _hashProvider = hashProvider; - _hashIdentities = hashIdentities; - } - - public ITargetBlock CreateBlock() - { - return new ActionBlock( - s => CopyXmlLookupsToResources(s.Source, s.Destination), - new ExecutionDataflowBlockOptions - { - BoundedCapacity = 1 - }); - } - - public void CopyXmlLookupsToResources(XmlReader reader, XmlWriter writer) - { - while (reader.Read()) - { - if (reader.NodeType == XmlNodeType.Element && _regex.IsMatch(reader.Name)) - { - using (var r = reader.ReadSubtree()) - { - var xElement = XElement.Parse(XElement.Load(r).ToString()); - ReplaceReferenceLookupsWithIdentities(xElement); - - using (var t2 = new StringReader(xElement.ToString())) - { - using (var r2 = new XmlTextReader(t2)) - { - while (r2.Read()) - { - WriteShallowNode(r2, writer, true); - } - } - } - } - - continue; - } - - WriteShallowNode(reader, writer); - } - - writer.Flush(); - } - - /// - /// Starting with the innermost nodes, replace all lookups with identities - /// - /// - private void ReplaceReferenceLookupsWithIdentities(XElement xElement) - { - XElement newIdentity = null; - - var ns = xElement.Name.Namespace; - - if (_regex.IsMatch(xElement.Name.LocalName)) - { - var typeName = _regex.Match(xElement.Name.LocalName).Groups["TypeName"].Value; - var lookup = xElement.Element(ns + $"{typeName}Lookup"); - var identity = xElement.Element(ns + $"{typeName}Identity"); - - if (lookup != null && identity == null) - { - var hashBytes = _hashProvider.Hash(lookup.ToString()); - var hash = _hashProvider.BytesToStr(hashBytes); - - if (_hashIdentities.ContainsKey(hash) && _hashIdentities[hash] != null) - { - newIdentity = XElement.Parse(_hashIdentities[hash].ToString()); - - lookup.AddBeforeSelf( - new XComment( - $"This {typeName}Lookup entity was used to populate its {typeName}Identity sibling")); - } - else - { - lookup.AddBeforeSelf( - new XComment( - $"No {typeName}Identity could be retrieved for this {typeName}Lookup entity")); - } - } - - foreach (var element in xElement.Elements()) - { - ReplaceReferenceLookupsWithIdentities(element); - } - - if (newIdentity == null) - { - return; - } - - xElement.Add(newIdentity); - - newIdentity.AddBeforeSelf( - new XComment( - $"This {typeName}Identity entity was retrieved from the API using the provided {typeName}Lookup information")); - } - else - { - foreach (var element in xElement.Elements()) - { - ReplaceReferenceLookupsWithIdentities(element); - } - } - } - - private static void WriteShallowNode(XmlReader reader, XmlWriter writer, bool skipXmlns = false) - { - switch (reader.NodeType) - { - case XmlNodeType.Element: - - if (skipXmlns) - { - writer.WriteStartElement(reader.LocalName); - - if (reader.HasAttributes) - { - reader.MoveToFirstAttribute(); - - do - { - if (reader.Name != "xmlns") - { - writer.WriteAttributeString(reader.Name, reader.Value); - } - } - while (reader.MoveToNextAttribute()); - } - } - else - { - writer.WriteStartElement(reader.Prefix, reader.LocalName, reader.NamespaceURI); - writer.WriteAttributes(reader, true); - } - - if (reader.IsEmptyElement) - { - writer.WriteEndElement(); - } - - break; - case XmlNodeType.Text: - writer.WriteString(reader.Value); - break; - case XmlNodeType.Whitespace: - case XmlNodeType.SignificantWhitespace: - writer.WriteWhitespace(reader.Value); - break; - case XmlNodeType.CDATA: - writer.WriteCData(reader.Value); - break; - case XmlNodeType.EntityReference: - writer.WriteEntityRef(reader.Name); - break; - case XmlNodeType.XmlDeclaration: - case XmlNodeType.ProcessingInstruction: - writer.WriteProcessingInstruction(reader.Name, reader.Value); - break; - case XmlNodeType.DocumentType: - - // ReSharper disable once AssignNullToNotNullAttribute - writer.WriteDocType( - reader.Name, reader.GetAttribute("PUBLIC"), reader.GetAttribute("SYSTEM"), - reader.Value); - - break; - case XmlNodeType.Comment: - writer.WriteComment(reader.Value); - break; - case XmlNodeType.EndElement: - writer.WriteFullEndElement(); - break; - } - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupWorkItem.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupWorkItem.cs deleted file mode 100644 index ea612220c0..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlLookupWorkItem.cs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class XmlLookupWorkItem - { - public XmlLookupWorkItem(XElement xelement) - { - LookupXElement = xelement; - } - - public string JsonModelName { get; set; } - - public string JsonResourceName { get; set; } - - public string ResourceName { get; set; } - - public string HashString { get; set; } - - public string IdentityName { get; set; } - - public string LookupName { get; set; } - - public XElement LookupXElement { get; } - - public XElement GetByExampleXElement { get; set; } - - public XElement ResourceXElement { get; set; } - - public XElement IdentityXElement { get; set; } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlToWorkItemsProcessor.cs b/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlToWorkItemsProcessor.cs deleted file mode 100644 index e719ddab5b..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/Engine/XmlLookupPipeline/XmlToWorkItemsProcessor.cs +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using System.Threading.Tasks.Dataflow; -using System.Xml; -using System.Xml.Linq; - -namespace EdFi.LoadTools.Engine.XmlLookupPipeline -{ - public class XmlToWorkItemsProcessor - { - private readonly Regex _regex = new Regex(Constants.ReferenceRegex); - - public IPropagatorBlock CreateBlock() - { - return new TransformManyBlock( - xr => GetLookupWorkItems(xr), - new ExecutionDataflowBlockOptions - { - BoundedCapacity = 1 - }); - } - - public IEnumerable GetLookupWorkItems(XmlReader reader) - { - while (reader.Read()) - { - if (reader.NodeType != XmlNodeType.Element || !_regex.IsMatch(reader.Name)) - { - continue; - } - - using (var r = reader.ReadSubtree()) - { - var xElement = XElement.Parse(XElement.Load(r).ToString()); - - foreach (var workItem in GetLookupWorkItems(xElement)) - { - yield return workItem; - } - } - } - } - - /// - /// - /// a ReferenceType xElement - /// A list of work items that should be looked up - private IEnumerable GetLookupWorkItems(XElement xElement) - { - var result = new List(); - var typeName = _regex.Match(xElement.Name.LocalName).Groups["TypeName"].Value; - var ns = xElement.Name.Namespace; - - // We only need to perform the lookup if they don't have an identity - var identity = xElement.Element(ns + $"{typeName}Identity"); - var lookup = xElement.Element(ns + $"{typeName}Lookup"); - - if (identity != null) - { - return GetLookupWorkItems(identity); - } - - if (lookup != null) - { - result.Add(new XmlLookupWorkItem(lookup)); - } - - result.AddRange( - xElement.Elements() - .Where(x => x.HasElements) - .SelectMany(GetLookupWorkItems) - ); - - return result; - } - } -} diff --git a/Utilities/DataLoading/EdFi.LoadTools/XmlLookupApplication.cs b/Utilities/DataLoading/EdFi.LoadTools/XmlLookupApplication.cs deleted file mode 100644 index cea92bda60..0000000000 --- a/Utilities/DataLoading/EdFi.LoadTools/XmlLookupApplication.cs +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Licensed to the Ed-Fi Alliance under one or more agreements. -// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. -// See the LICENSE and NOTICES files in the project root for more information. - -using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; -using System.Xml; -using EdFi.LoadTools.Engine; -using EdFi.LoadTools.Engine.XmlLookupPipeline; - -namespace EdFi.LoadTools -{ - public class XmlLookupApplication - { - private readonly IXmlPairsFactory _pairsFactory; - private readonly XmlLookupPipelineProcessor _xmlLookupPipelineProcessor; - private readonly XmlLookupToResourceProcessor _xmlLookupToResourceProcessor; - private readonly XmlToWorkItemsProcessor _xmlToWorkItemsProcessor; - - public XmlLookupApplication( - IXmlPairsFactory pairsFactory, - XmlToWorkItemsProcessor xmlToWorkItemsProcessor, - XmlLookupPipelineProcessor xmlLookupPipelineProcessor, - XmlLookupToResourceProcessor xmlLookupToResourceProcessor) - { - _pairsFactory = pairsFactory; - _xmlToWorkItemsProcessor = xmlToWorkItemsProcessor; - _xmlLookupPipelineProcessor = xmlLookupPipelineProcessor; - _xmlLookupToResourceProcessor = xmlLookupToResourceProcessor; - } - - public async Task Run() - { - // find and resolve lookups to references - var resolveLookupsPipeline = ResolveLookupsPipeline(); - - foreach (var reader in _pairsFactory.GetSources()) - { - await resolveLookupsPipeline.StartBlock.SendAsync(reader).ConfigureAwait(false); - } - - resolveLookupsPipeline.StartBlock.Complete(); - await resolveLookupsPipeline.Completion.ConfigureAwait(false); - - // write new xml file with references - var writeUpdatedXmlPipeline = WriteUpdatedXmlPipeline(); - - foreach (var streamPair in _pairsFactory.GetIoPairs()) - { - await writeUpdatedXmlPipeline.StartBlock.SendAsync(streamPair).ConfigureAwait(false); - } - - writeUpdatedXmlPipeline.StartBlock.Complete(); - await writeUpdatedXmlPipeline.Completion.ConfigureAwait(false); - - return 0; - } - - private DataFlowPipeline ResolveLookupsPipeline() - { - var lookupBlock = _xmlToWorkItemsProcessor.CreateBlock(); - var lookupBuffer = new BufferBlock(); - var lookupPipeline = _xmlLookupPipelineProcessor.CreateBlock(); - - lookupBlock.LinkTo( - lookupBuffer, new DataflowLinkOptions - { - PropagateCompletion = true - }); - - lookupBuffer.LinkTo( - lookupPipeline, new DataflowLinkOptions - { - PropagateCompletion = true - }); - - return new DataFlowPipeline(lookupBlock, lookupPipeline.Completion); - } - - private DataFlowPipeline WriteUpdatedXmlPipeline() - { - var copyBuffer = new BufferBlock(); - var copyXmlBlock = _xmlLookupToResourceProcessor.CreateBlock(); - - copyBuffer.LinkTo( - copyXmlBlock, new DataflowLinkOptions - { - PropagateCompletion = true - }); - - return new DataFlowPipeline(copyBuffer, copyXmlBlock.Completion); - } - } -} diff --git a/Utilities/DataLoading/LoadTools.sln b/Utilities/DataLoading/LoadTools.sln index d2405cffe7..cad6cc5572 100644 --- a/Utilities/DataLoading/LoadTools.sln +++ b/Utilities/DataLoading/LoadTools.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.LoadTools.Test", "EdFi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.LoadTools", "EdFi.LoadTools\EdFi.LoadTools.csproj", "{DE1E614A-422F-4109-AEA9-6E4D2982E182}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.XmlLookup.Console", "EdFi.XmlLookup.Console\EdFi.XmlLookup.Console.csproj", "{7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.SmokeTest.Console", "EdFi.SmokeTest.Console\EdFi.SmokeTest.Console.csproj", "{3C4DCD4B-862A-45EA-AC93-E91EE357D2A6}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.BulkLoadClient.Console", "EdFi.BulkLoadClient.Console\EdFi.BulkLoadClient.Console.csproj", "{DA7A7FC0-FE00-43AB-8F2D-6DFFE89C7BA9}" @@ -37,14 +35,6 @@ Global {DE1E614A-422F-4109-AEA9-6E4D2982E182}.Release|Any CPU.Build.0 = Release|Any CPU {DE1E614A-422F-4109-AEA9-6E4D2982E182}.Release|x64.ActiveCfg = Release|Any CPU {DE1E614A-422F-4109-AEA9-6E4D2982E182}.Release|x64.Build.0 = Release|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Debug|x64.ActiveCfg = Debug|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Debug|x64.Build.0 = Debug|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Release|Any CPU.Build.0 = Release|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Release|x64.ActiveCfg = Release|Any CPU - {7EB8EA5F-8626-46A5-AE69-7DA7E50678B5}.Release|x64.Build.0 = Release|Any CPU {3C4DCD4B-862A-45EA-AC93-E91EE357D2A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3C4DCD4B-862A-45EA-AC93-E91EE357D2A6}.Debug|Any CPU.Build.0 = Debug|Any CPU {3C4DCD4B-862A-45EA-AC93-E91EE357D2A6}.Debug|x64.ActiveCfg = Debug|Any CPU