Dataset Management API Support

Net45.SODA.Tests/Net45.SODA.Tests.csproj

@@ -9,9 +9,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="nunit" Version="3.10.1" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.10.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" />
+    <PackageReference Include="nunit" Version="3.12.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.15.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
   </ItemGroup>
 
   <ItemGroup>

README.md

@@ -1,6 +1,6 @@
 # SODA.NET [![Build status](https://ci.appveyor.com/api/projects/status/yub6lyl79573lufv/branch/master?svg=true)](https://ci.appveyor.com/project/thekaveman/soda-net/branch/master)
 
-A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting 
+A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting
 .NET 4.5 and above.
 
 ## Getting Started
@@ -47,7 +47,7 @@ var results = dataset.Query<MyOtherClass>(soql);
 
 ```c#
 //make sure to provide auth credentials!
-var client = 
+var client =
     new SodaClient("data.smgov.net", "AppToken", "[email protected]", "password");
 
 //Upsert some data serialized as CSV
@@ -59,9 +59,113 @@ IEnumerable<MyClass> payload = GetPayloadData();
 client.Upsert(payload, "1234-wxyz");
 ```
 
-Note: This library supports writing directly to datasets with the Socrata Open Data API. For write operations that use 
-data transformations in the Socrata Data Management Experience (the user interface for creating datasets), use the Socrata 
-Data Management API. For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#)
+**SodaClient** cam also be used for performing Dataset Management API requests
+
+For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#)
+
+Creating datasets:
+```c#
+using System;
+using SODA;
+using System.Diagnostics;
+
+namespace SocrataTest
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            // Initialize the client
+            SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}");
+
+            // Read in File (or other source)
+            string filepath = "C:\\Users\\{user}\\Desktop\\test.csv";
+            string csv = System.IO.File.ReadAllText(filepath);
+
+            // Create a Dataset - either public or private (default: private)
+            Revision dataset = pipelineClient.CreateDataset("MyNewDataset", "public");
+
+            string datasetId = dataset.GetFourFour();
+            Console.WriteLine(datasetId);
+
+            Source source = pipelineClient.CreateSource(csv, dataset, SodaDataFormat.CSV, "File");
+            SchemaTransforms input = pipelineClient.CreateInputSchema(source);
+            AppliedTransform output = input.Run();
+            output.AwaitCompletion(pipelineClient, status => { });
+
+            // Check for Errors
+            if (output.GetErrorCount() > 0)
+            {
+                Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount()));
+                pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output);
+                // Optional Throw new Error...
+            }
+
+            // Apply the revision to the dataset
+            PipelineJob job = pipelineClient.Apply(output, dataset);
+
+            // Await the completion of the revision and output the processing log
+            job.AwaitCompletion(status => Console.WriteLine(status));
+
+        }
+    }
+}
+```
+
+Creating update, replace, or delete revisions:
+```cs
+using System;
+using SODA;
+using System.Diagnostics;
+
+namespace SocrataTest
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            // Initialize the client
+            SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}");
+
+            // Read in File (or other source)
+            string filepath = "C:\\Users\\{user}\\Desktop\\test.csv";
+            string csv = System.IO.File.ReadAllText(filepath);
+
+            // CREATING A REVISION
+            // Create a Revision (either update, replace, or delete)
+            Revision revision = pipelineClient.CreateRevision("update", "1234-abcd");
+
+            // Upload the file as a new source
+            Source newSource = pipelineClient.CreateSource(csv, revision, SodaDataFormat.CSV, "MyNewFile");
+            //Console.WriteLine(source.GetSchemaId());
+            // Get the schema of the new (latest) source
+            SchemaTransforms newInput = pipelineClient.CreateInputSchema(newSource);
+
+            // Run the output transforms
+            AppliedTransform newOutput = newInput.Run();
+
+            // Transforms are applied asynchronously, so we need to wait for them to complete
+            newOutput.AwaitCompletion(pipelineClient, status => { });
+
+            // Check for Errors
+            if(output.GetErrorCount() > 0)
+            {
+                Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount()));
+                pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output);
+                // Optional Throw new Error...
+            }
+
+            // Apply the revision to replace/update the dataset
+            PipelineJob newJob = pipelineClient.Apply(newOutput, revision);
+
+            // Await the completion of the revision and output the processing log
+            newJob.AwaitCompletion(status => Console.WriteLine(status) );
+
+        }
+    }
+}
+
+```
 
 ## Build
 
@@ -84,15 +188,15 @@ To create the Nuget package artifacts, pass an extra parameter:
 
 ## Contributing
 
-Check out the 
-[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) 
+Check out the
+[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md)
 for more details.
 
 ## Copyright and License
 
 Copyright 2017 City of Santa Monica, CA
 
-Licensed under the 
+Licensed under the
 [MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt)
 
 ## Thank you

SODA.Tests/SodaClientTests.cs

@@ -266,5 +266,37 @@ public void DeleteRow_Using_Anonymous_Client_Throws_InvalidOperationException()
         {
             Assert.That(() => mockClient.DeleteRow(StringMocks.NonEmptyInput, StringMocks.ResourceId), Throws.TypeOf<InvalidOperationException>());
         }
+
+        [TestCase(StringMocks.NullInput)]
+        [TestCase(StringMocks.EmptyInput)]
+        [Category("SodaClient")]
+        public void DSMAPI_With_Invalid_ResourceId_Throws_ArgumentOutOfRangeException(string input)
+        {
+            Assert.That(() => mockClient.CreateRevision(StringMocks.NonEmptyInput, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+        }
+
+        [TestCase(StringMocks.NullInput)]
+        [TestCase(StringMocks.EmptyInput)]
+        [Category("SodaClient")]
+        public void DSMAPI_With_NULL_Or_Empty_Values_Throws_ArgumentOutOfRangeException(string input)
+        {
+            Assert.That(() => mockClient.CreateDataset(input), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => mockClient.CreateRevision(input, StringMocks.ResourceId), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => mockClient.CreateSource(input, null, SodaDataFormat.CSV), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => mockClient.ExportErrorRows(input, null), Throws.TypeOf<ArgumentException>());
+
+        }
+
+        [Category("SodaClient")]
+        public void DSMAPI_With_NUL_Values_Throws_ArgumentOutOfRangeException()
+        {
+            Assert.That(() => mockClient.CreateSource(StringMocks.NonEmptyInput, null, SodaDataFormat.CSV), Throws.TypeOf<ArgumentException>());
+
+            Assert.That(() => mockClient.GetSource(null), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => mockClient.CreateInputSchema(null), Throws.TypeOf<ArgumentException>());
+
+            Assert.That(() => mockClient.ExportErrorRows(StringMocks.NonEmptyInput, null), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => mockClient.Apply(null, null), Throws.TypeOf<InvalidOperationException>());
+        }
     }
 }

SODA.Tests/SodaUriTests.cs

@@ -38,6 +38,30 @@ public void All_Methods_Return_Uri_With_Socrata_Domain_As_Host()
             uri = null;
             uri = SodaUri.ForCategoryPage(StringMocks.Host, StringMocks.NonEmptyInput);
             StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForRevision(StringMocks.Host, StringMocks.ResourceId);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForSource(StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForUpload(StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForApply(StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForJob(StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
+
+            uri = null;
+            uri = SodaUri.ForErrorRows(StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host);
         }
 
         [Test]
@@ -98,6 +122,30 @@ public void All_Methods_Return_Uri_Using_HTTPS()
             uri = null;
             uri = SodaUri.ForCategoryPage("http://" + StringMocks.Host, StringMocks.NonEmptyInput);
             StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForRevision("http://" + StringMocks.Host, StringMocks.ResourceId);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForSource("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForUpload("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForApply("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForJob("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
+
+            uri = null;
+            uri = SodaUri.ForErrorRows("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput);
+            StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme);
         }
 
         [TestCase(StringMocks.NullInput)]
@@ -328,6 +376,30 @@ public void ForCategoryPage_With_Empty_Category_Throws_ArgumentException(string
             Assert.That(() => SodaUri.ForCategoryPage(StringMocks.Host, input), Throws.TypeOf<ArgumentException>());
         }
 
+        [TestCase(StringMocks.NullInput)]
+        [TestCase(StringMocks.EmptyInput)]
+        [Category("SodaUri")]
+        public void DSMAPI_With_Invalid_Endpoint_Throws_ArgumentOutOfRangeException(string input)
+        {
+            Assert.That(() => SodaUri.ForRevision(StringMocks.Host, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+            Assert.That(() => SodaUri.ForUpload(StringMocks.Host, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+            Assert.That(() => SodaUri.ForSource(StringMocks.Host, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+            Assert.That(() => SodaUri.ForJob(StringMocks.Host, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+            Assert.That(() => SodaUri.ForApply(StringMocks.Host, input), Throws.TypeOf<ArgumentOutOfRangeException>());
+        }
+
+        [TestCase(StringMocks.NullInput)]
+        [TestCase(StringMocks.EmptyInput)]
+        [Category("SodaUri")]
+        public void DSMAPI_With_Invalid_Host_Throws_ArgumentOutOfRangeException(string input)
+        {
+            Assert.That(() => SodaUri.ForRevision(input, StringMocks.NonEmptyInput), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => SodaUri.ForUpload(input, StringMocks.NonEmptyInput), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => SodaUri.ForSource(input, StringMocks.NonEmptyInput), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => SodaUri.ForJob(input, StringMocks.NonEmptyInput), Throws.TypeOf<ArgumentException>());
+            Assert.That(() => SodaUri.ForApply(input, StringMocks.NonEmptyInput), Throws.TypeOf<ArgumentException>());
+        }
+
         [Test]
         [Category("SodaUri")]
         public void ForCategoryPage_With_Valid_Arguments_Creates_CategoryPage_Uri()
@@ -352,4 +424,4 @@ public void ForCategoryPage_With_Complex_Category_Uri_Doesnt_Escape_Complex_Cate
             StringAssert.AreNotEqualIgnoringCase(String.Format("/categories/{0}", Uri.EscapeDataString(complexCategory)), uri.LocalPath);
         }
     }
-}
+}

SODA/AppliedTransform.cs

@@ -0,0 +1,77 @@
+using System.Collections.Generic;
+using System;
+using SODA.Utilities;
+
+namespace SODA
+{
+    /// <summary>
+    /// A class for Applying Transforms.
+    /// </summary>
+    public class AppliedTransform
+    {
+        /// <summary>
+        /// Source object.
+        /// </summary>
+        Source source;
+
+        /// <summary>
+        /// Create an applied transform object based off a source.
+        /// </summary>
+        /// <returns>AppliedTransform</returns>
+        public AppliedTransform(Source source)
+        {
+            this.source = source;
+        }
+
+        /// <summary>
+        /// Retrieve the Output schema id.
+        /// </summary>
+        /// <returns>Error count</returns>
+        public string GetOutputSchemaId()
+        {
+            return this.source.GetSchemaId();
+        }
+
+        /// <summary>
+        /// Retrieve Input schema ID.
+        /// </summary>
+        /// <returns>Error count</returns>
+        public string GetInputSchemaId()
+        {
+            return this.source.GetInputSchemaId();
+        }
+
+        /// <summary>
+        /// Retrieve the error count.
+        /// </summary>
+        /// <returns>Error count</returns>
+        public int GetErrorCount()
+        {
+            return this.source.GetErrorCount();
+        }
+
+        /// <summary>
+        /// Retrieve the error endpoint.
+        /// </summary>
+        /// <returns>Error endpoint</returns>
+        public string GetErrorRowEndpoint()
+        {
+            return this.source.GetErrorRowEndPoint();
+        }
+
+        /// <summary>
+        /// Await completion of transforms.
+        /// </summary>
+        /// <param name="client">The current Soda client</param>
+        /// <param name="lambda">Lambda output</param>
+        public void AwaitCompletion(SodaClient client, Action<string> lambda)
+        {
+            this.source = client.GetSource(this.source);
+            while (!this.source.IsComplete(lambda))
+            {
+                this.source = client.GetSource(this.source);
+                System.Threading.Thread.Sleep(1000);
+            }        
+        }
+    }
+}