From 8b6b8158dd139cacd9d6c9bce455acd64229b1b0 Mon Sep 17 00:00:00 2001 From: peter Moore Date: Fri, 4 Oct 2019 15:25:23 -0700 Subject: [PATCH 1/8] initial framework for DSMAPI .NET support --- README.md | 59 ++++++++-- SODA/AppliedTransform.cs | 23 ++++ SODA/PipelineJob.cs | 58 ++++++++++ SODA/Revision.cs | 52 +++++++++ SODA/SchemaTransforms.cs | 78 +++++++++++++ SODA/SodaDSMAPIClient.cs | 238 ++++++++++++++++++++++++++++++++++++++ SODA/Source.cs | 57 +++++++++ SODA/Utilities/SodaUri.cs | 64 +++++++++- 8 files changed, 619 insertions(+), 10 deletions(-) create mode 100644 SODA/AppliedTransform.cs create mode 100644 SODA/PipelineJob.cs create mode 100644 SODA/Revision.cs create mode 100644 SODA/SchemaTransforms.cs create mode 100644 SODA/SodaDSMAPIClient.cs create mode 100644 SODA/Source.cs diff --git a/README.md b/README.md index 0af98ec..8d159b1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SODA.NET [![Build status](https://ci.appveyor.com/api/projects/status/yub6lyl79573lufv/branch/master?svg=true)](https://ci.appveyor.com/project/thekaveman/soda-net/branch/master) -A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting +A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting .NET 4.5 and above. ## Getting Started @@ -47,7 +47,7 @@ var results = dataset.Query(soql); ```c# //make sure to provide auth credentials! -var client = +var client = new SodaClient("data.smgov.net", "AppToken", "user@domain.com", "password"); //Upsert some data serialized as CSV @@ -59,9 +59,52 @@ IEnumerable payload = GetPayloadData(); client.Upsert(payload, "1234-wxyz"); ``` -Note: This library supports writing directly to datasets with the Socrata Open Data API. For write operations that use -data transformations in the Socrata Data Management Experience (the user interface for creating datasets), use the Socrata -Data Management API. For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#) +**SodaDSMAPIClient** is used for performing Dataset Management API requests + +For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#) + +```c# +using System; +using SODA; + +namespace MyApp +{ + class MyETLClass + { + static void Main(string[] args) + { + // Initialize the client + PipelineClient pipelineClient = new SodaDSMAPIClient("https://my.data.socrata.com", "username", "password"); + + // Read in File (or other source) + string filepath = "C:\\Users\\mysource\\outputs\\output.csv"; + string csv = System.IO.File.ReadAllText(filepath); + + // Create a REVISION + Revision revision = pipelineClient.CreateRevision("replace", "1234-abcd"); + + // Upload the file as a new source + Source source = pipelineClient.CreateSource(csv, revision, DataFormat.CSV, filename="MyNewFile") + + // Await data upload + source.AwaitCompletion() + + // Get the schema of the new (latest) source + SchemaTransforms input = pipelineClient.CreateInputSchema(source); + + // Do transforms + // AppliedTransforms output = input.ChangeColumnDisplayName("oldname","newname").ChangeColumnDescription("newname","New description").Run(); + // + + // Apply the revision to replace/update the dataset + PipelineJob job = pipelineClient.Apply(input, revision); + + // Await the completion of the revision and output the processing log + job.AwaitCompletion(); + } + } +} +``` ## Build @@ -84,15 +127,15 @@ To create the Nuget package artifacts, pass an extra parameter: ## Contributing -Check out the -[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) +Check out the +[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) for more details. ## Copyright and License Copyright 2017 City of Santa Monica, CA -Licensed under the +Licensed under the [MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt) ## Thank you diff --git a/SODA/AppliedTransform.cs b/SODA/AppliedTransform.cs new file mode 100644 index 0000000..2436bca --- /dev/null +++ b/SODA/AppliedTransform.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using SODA.Utilities; + +namespace SODA +{ + /// + /// A class for Applying Transforms. + /// + public class AppliedTransform + { + Dictionary resource; + public AppliedTransform(Dictionary transform) + { + + } + public string GetOutputSchemaId() + { + return this.resource["schemas"][0]["output_schemas"][0]["id"]; + } + } +} diff --git a/SODA/PipelineJob.cs b/SODA/PipelineJob.cs new file mode 100644 index 0000000..e1c0426 --- /dev/null +++ b/SODA/PipelineJob.cs @@ -0,0 +1,58 @@ +using System; +using SODA.Utilities; +using System.Net; + +namespace SODA +{ + /// + /// A class representing the Data Pipeline job for revisions that have been submitted to Socrata. + /// + public class PipelineJob + { + public string Username; + private string password; + public Uri revisionEndpoint { get; set; } + + public PipelineJob(Uri revEndpoint, string user, string pass, long revNum) + { + Username = user; + password = pass; + + revisionEndpoint = SocrataUri.ForJob(revEndpoint, revNum); + Console.WriteLine(revisionEndpoint); + var jobRequest = new SodaRequest(revisionEndpoint, "GET", Username, password, DataFormat.JSON); + Result r = null; + try + { + r = jobRequest.ParseResponse(); + Console.WriteLine(r.Resource["task_sets"][0]["status"]); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + r = new Result() { Message = webException.Message, IsError = true, ErrorCode = message }; + } + } + public void AwaitCompletion() + { + string status = ""; + Result r = null; + while(status != "successful" && status != "failure") + { + var jobRequest = new SodaRequest(revisionEndpoint, "GET", Username, password, DataFormat.JSON); + try + { + r = jobRequest.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + r = new Result() { Message = webException.Message, IsError = true, ErrorCode = message }; + } + status = r.Resource["task_sets"][0]["status"]; + Console.WriteLine(status); + System.Threading.Thread.Sleep(1000); + } + } + } +} diff --git a/SODA/Revision.cs b/SODA/Revision.cs new file mode 100644 index 0000000..5d540eb --- /dev/null +++ b/SODA/Revision.cs @@ -0,0 +1,52 @@ +using System; +using SODA.Utilities; +using System.Security.Permissions; +using NLog; + +namespace SODA + +{ + /// + /// A class for accessing the revision object. + /// + public class Revision + { + Logger logger = LogManager.GetCurrentClassLogger(); + /// + /// The result of a revision being created. + /// + Result result; + + /// + /// A class for handling revisions. + /// + public Revision(Result result) + { + this.result = result; + logger.Info(String.Format("Revision number {0} created", result.Resource["revision_seq"])); + + } + + public long GetRevisionNumber() + { + return this.result.Resource["revision_seq"]; + } + + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public string GetSourceEndpoint() + { + return this.result.Links["create_source"]; + } + + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public string GetApplyEndpoint() + { + return this.result.Links["apply"]; + } + + } +} diff --git a/SODA/SchemaTransforms.cs b/SODA/SchemaTransforms.cs new file mode 100644 index 0000000..20f2ddb --- /dev/null +++ b/SODA/SchemaTransforms.cs @@ -0,0 +1,78 @@ +using System.Collections.Generic; +using System; +namespace SODA +{ + /// + /// A class for applying transforms to the schema programmatically. + /// + public class SchemaTransforms + { + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + Dictionary source; + + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public SchemaTransforms(Source source) + { + this.source = source.Resource; + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnFieldname() + { + + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnDescription() + { + + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnDisplayName() + { + + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnTransform() + { + + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void AddColumn() + { + + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public Dictionary GetSource() + { + return this.source; + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public string GetOutputSchemaId() + { + return this.source["schemas"][0]["output_schemas"][0]["id"]; + } + + public AppliedTransform Run() + { + return new AppliedTransform(this.source); + } + + } +} diff --git a/SODA/SodaDSMAPIClient.cs b/SODA/SodaDSMAPIClient.cs new file mode 100644 index 0000000..aeafbaf --- /dev/null +++ b/SODA/SodaDSMAPIClient.cs @@ -0,0 +1,238 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using SODA.Utilities; + +namespace SODA +{ + /// + /// A class for interacting with Socrata Data Portals using the Socrata Dataset Management API. + /// + public class SodaDSMAPIClient + { + /// + /// The url to the Socrata Open Data Portal this client targets. + /// + public readonly string Host; + + /// + /// The user account that this client uses for Authentication during each request. + /// + /// + /// Authentication is only necessary when accessing datasets that have been marked as private or when making write requests (PUT, POST, and DELETE). + /// See http://dev.socrata.com/docs/authentication.html for more information. + /// + public readonly string Username; + + //not publicly readable, can only be set in a constructor + private readonly string password; + + /// + /// If set, the number of milliseconds to wait before requests to the timeout and throw a . + /// If unset, the default value is that of . + /// + public int? RequestTimeout { get; set; } + + /// + /// Initialize a new SodaClient for the specified Socrata host, using the specified application token and Authentication credentials. + /// + /// The Socrata Open Data Portal that this client will target. + /// The Socrata application token that this client will use for all requests. + /// The user account that this client will use for Authentication during each request. + /// The password for the specified that this client will use for Authentication during each request. + /// Thrown if no is provided. + public SodaDSMAPIClient(string host, string appToken, string username, string password) + { + if (String.IsNullOrEmpty(host)) + throw new ArgumentException("host", "A host is required"); + + Host = SodaUri.enforceHttps(host); + AppToken = appToken; + Username = username; + this.password = password; + } + + /// + /// Initialize a new SodaClient for the specified Socrata host, using the specified Authentication credentials. + /// + /// The Socrata Open Data Portal that this client will target. + /// The user account that this client will use for Authentication during each request. + /// The password for the specified that this client will use for Authentication during each request. + /// Thrown if no is provided. + public SodaDSMAPIClient(string host, string username, string password) + : this(host, null, username, password) + { + } + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// One of upsert, replace, or delete + /// The identifier (4x4) for a resource on the Socrata host to target. + /// The permission level of the dataset, can be one of either "public" or "private". + /// A newly created Revision. + /// Thrown if the specified does not match the Socrata 4x4 pattern. + public Revision CreateRevision(string method, string resourceId, string permission = "private") + { + if (FourByFour.IsNotValid(resourceId)) + throw new ArgumentOutOfRangeException("The provided resourceId is not a valid Socrata (4x4) resource identifier.", nameof(resourceId)); + + revisionUri = SodaUri.ForRevision(Host, resourceId); + var dataFormat = DataFormat.JSON; + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); + action["type"] = method; + action["permission"] = permission; + payload["action"] = action; + + logger.Info(revisionUri); + Console.WriteLine(revisionUri); + var request = new SodaRequest(revisionUri, "POST", Username, password, dataFormat, payload.ToString()); + + Result result = null; + try + { + result = request.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new Revision(result); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// A string of serialized data. + /// The revision created as part of a create revision step. + /// The format of the data. + /// The filename that should be associated with this upload. + /// A indicating success or failure. + /// Thrown if this SodaDSMAPIClient was initialized without authentication credentials. + public Source CreateSource(string data, Revision revision, DataFormat dataFormat, string filename = "NewUpload") + { + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + var uri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); + + revisionNumber = revision.GetRevisionNumber(); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject(); + source_type["type"] = "upload"; + source_type["filename"] = filename; + payload["source_type"] = source_type; + + logger.Info(uri); + Console.WriteLine(uri); + var createSourceRequest = new SodaRequest(uri, "POST", Username, password, DataFormat.JSON, payload.ToString()); + Source source = null; + try + { + source = createSourceRequest.ParseResponse(); + string uploadDataPath = source.Links["bytes"]; + uri = SodaUri.ForUpload(Host, uploadDataPath); + Console.WriteLine(uri); + + var fileUploadRequest = new SodaRequest(uri, "POST", Username, password, dataFormat, data); + source = fileUploadRequest.ParseResponse(); + + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + source = new Source() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + source = new Source() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return source; + } + + + /// + /// Create the InputSchema from the source. + /// + /// The result of the Source creation + /// A SchemaTransforms object + public SchemaTransforms CreateInputSchema(Source source) + { + return new SchemaTransforms(source); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// A string of serialized data. + /// A indicating success or failure. + public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) + { + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); + + var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); + logger.Info(uri); + Console.WriteLine(uri); + var applyRequest = new SodaRequest(uri, "PUT", Username, password, DataFormat.JSON, payload.ToString()); + Result result = null; + try + { + result = applyRequest.ParseResponse(); + + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new PipelineJob(revisionUri, Username, password, revisionNumber); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// A string of serialized data. + /// A string of serialized data. + /// A indicating success or failure. + public PipelineJob Apply(SchemaTransforms inputSchema, Revision revision) + { + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + payload["output_schema_id"] = inputSchema.GetOutputSchemaId(); + + var uri = SodaUri.ForApply(Host, revision.GetApplyEndpoint()); + logger.Info(uri); + Console.WriteLine(uri); + var applyRequest = new SodaRequest(uri, "PUT", Username, password, DataFormat.JSON, payload.ToString()); + Result result = null; + try + { + result = applyRequest.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new PipelineJob(revisionUri, Username, password, revisionNumber); + } + } +} diff --git a/SODA/Source.cs b/SODA/Source.cs new file mode 100644 index 0000000..d34b013 --- /dev/null +++ b/SODA/Source.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using SODA.Utilities; + +namespace SODA +{ + /// + /// Gets the information related to the Source. + /// + [DataContract] + public class Source + { + /// + /// Gets the number of errors. + /// + [DataMember(Name = "Errors")] + public int Errors { get; private set; } + + /// + /// Gets the explanatory text about this result. + /// + [DataMember(Name = "message")] + public string Message { get; internal set; } + + /// + /// Gets a flag indicating if one or more errors occured. + /// + [DataMember(Name = "error")] + public bool IsError { get; internal set; } + + /// + /// Gets data about any errors that occured. + /// + [DataMember(Name = "code")] + public string ErrorCode { get; internal set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "data")] + public dynamic Data { get; internal set; } + + /// + /// Get data related to the resource. + /// + [DataMember(Name = "resource")] + public Dictionary Resource { get; set; } + + /// + /// Gets links provided for gathering additional resources. + /// + [DataMember(Name = "links")] + public Dictionary Links { get; set; } + } +} diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index 82dba3b..e5d0c9e 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -203,7 +203,7 @@ public static Uri ForQuery(string socrataHost, string resourceId, SoqlQuery soql return new Uri(queryUrl); } - + /// /// Create a Uri to the landing page of a specified category on the specified Socrata host. /// @@ -222,5 +222,65 @@ public static Uri ForCategoryPage(string socrataHost, string category) return new Uri(url); } + /// + /// Create a revision Uri the specified resource on the specified Socrata host. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A revision Uri for the specified resource on the specified Socrata host. + public static Uri ForRevision(string socrataHost, string resourceId) + { + string url = String.Format("{0}/api/publishing/v1/revision/{1}", socrataHost, resourceId); + return new Uri(url); + } + + /// + /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForSource(string socrataHost, string sourceEndpoint) + { + string url = String.Format("{0}{1}", socrataHost, sourceEndpoint.Replace("\"", "")); + return new Uri(url); + + } + + /// + /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForUpload(string socrataHost, string uploadEndpoint) + { + string url = String.Format("{0}{1}", socrataHost, uploadEndpoint.Replace("\"", "")); + return new Uri(url); + } + + /// + /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForApply(string socrataHost, string applyEndpoint) + { + string url = String.Format("{0}{1}", socrataHost, applyEndpoint.Replace("\"", "")); + return new Uri(url); + } + + /// + /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForJob(Uri revisionEndpoint, long revisionNumber) + { + string url = String.Format("{0}/{1}/", revisionEndpoint.ToString(), revisionNumber); + return new Uri(url); + } } -} \ No newline at end of file +} From 16f3fbf6e9eee940872260b773b6d7ae18c36ae5 Mon Sep 17 00:00:00 2001 From: peter Moore Date: Tue, 8 Oct 2019 09:11:46 -0700 Subject: [PATCH 2/8] move back into client --- Net45.SODA.Tests/Net45.SODA.Tests.csproj | 6 +- SODA.Tests/SodaUriTests.cs | 18 +- SODA/PipelineJob.cs | 8 +- SODA/Result.cs | 59 ++++++ SODA/Revision.cs | 9 +- SODA/SodaClient.cs | 139 ++++++++++++- SODA/SodaDSMAPIClient.cs | 238 ----------------------- SODA/Utilities/SodaUri.cs | 52 +++-- 8 files changed, 265 insertions(+), 264 deletions(-) create mode 100644 SODA/Result.cs delete mode 100644 SODA/SodaDSMAPIClient.cs diff --git a/Net45.SODA.Tests/Net45.SODA.Tests.csproj b/Net45.SODA.Tests/Net45.SODA.Tests.csproj index c08b6f0..950f67c 100644 --- a/Net45.SODA.Tests/Net45.SODA.Tests.csproj +++ b/Net45.SODA.Tests/Net45.SODA.Tests.csproj @@ -9,9 +9,9 @@ - - - + + + diff --git a/SODA.Tests/SodaUriTests.cs b/SODA.Tests/SodaUriTests.cs index 3b32cc9..d654c38 100644 --- a/SODA.Tests/SodaUriTests.cs +++ b/SODA.Tests/SodaUriTests.cs @@ -38,6 +38,22 @@ public void All_Methods_Return_Uri_With_Socrata_Domain_As_Host() uri = null; uri = SodaUri.ForCategoryPage(StringMocks.Host, StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForRevision(StringMocks.Host, StringMocks.ResourceId); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForSource(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForApply(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForJob(StringMocks.Host, 1); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); } [Test] @@ -352,4 +368,4 @@ public void ForCategoryPage_With_Complex_Category_Uri_Doesnt_Escape_Complex_Cate StringAssert.AreNotEqualIgnoringCase(String.Format("/categories/{0}", Uri.EscapeDataString(complexCategory)), uri.LocalPath); } } -} \ No newline at end of file +} diff --git a/SODA/PipelineJob.cs b/SODA/PipelineJob.cs index e1c0426..2198a49 100644 --- a/SODA/PipelineJob.cs +++ b/SODA/PipelineJob.cs @@ -13,14 +13,14 @@ public class PipelineJob private string password; public Uri revisionEndpoint { get; set; } - public PipelineJob(Uri revEndpoint, string user, string pass, long revNum) + public PipelineJob(string revEndpoint, string user, string pass, long revNum) { Username = user; password = pass; - revisionEndpoint = SocrataUri.ForJob(revEndpoint, revNum); + revisionEndpoint = SodaUri.ForJob(revEndpoint, revNum); Console.WriteLine(revisionEndpoint); - var jobRequest = new SodaRequest(revisionEndpoint, "GET", Username, password, DataFormat.JSON); + var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, Username, password, SodaDataFormat.JSON); Result r = null; try { @@ -39,7 +39,7 @@ public void AwaitCompletion() Result r = null; while(status != "successful" && status != "failure") { - var jobRequest = new SodaRequest(revisionEndpoint, "GET", Username, password, DataFormat.JSON); + var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, Username, password, SodaDataFormat.JSON); try { r = jobRequest.ParseResponse(); diff --git a/SODA/Result.cs b/SODA/Result.cs new file mode 100644 index 0000000..7b9ecfd --- /dev/null +++ b/SODA/Result.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using SODA.Utilities; + +namespace SODA +{ + /// + /// Gets the information related to the Result of API calls. + /// + [DataContract] + public class Result + { + /// + /// Gets the number of errors. + /// + [DataMember(Name = "Errors")] + public int Errors { get; private set; } + + + /// + /// Gets the explanatory text about this result. + /// + [DataMember(Name = "message")] + public string Message { get; internal set; } + + /// + /// Gets a flag indicating if one or more errors occured. + /// + [DataMember(Name = "error")] + public bool IsError { get; internal set; } + + /// + /// Gets data about any errors that occured. + /// + [DataMember(Name = "code")] + public string ErrorCode { get; internal set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "data")] + public dynamic Data { get; internal set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "resource")] + public Dictionary Resource { get; set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "links")] + public Dictionary Links { get; set; } + + } +} diff --git a/SODA/Revision.cs b/SODA/Revision.cs index 5d540eb..72817e5 100644 --- a/SODA/Revision.cs +++ b/SODA/Revision.cs @@ -1,7 +1,6 @@ using System; using SODA.Utilities; using System.Security.Permissions; -using NLog; namespace SODA @@ -11,7 +10,6 @@ namespace SODA /// public class Revision { - Logger logger = LogManager.GetCurrentClassLogger(); /// /// The result of a revision being created. /// @@ -23,7 +21,7 @@ public class Revision public Revision(Result result) { this.result = result; - logger.Info(String.Format("Revision number {0} created", result.Resource["revision_seq"])); + Console.WriteLine(String.Format("Revision number {0} created", result.Resource["revision_seq"])); } @@ -32,6 +30,11 @@ public long GetRevisionNumber() return this.result.Resource["revision_seq"]; } + public string getRevisionLink() + { + return this.result.Links["self"]; + } + /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// diff --git a/SODA/SodaClient.cs b/SODA/SodaClient.cs index 9b26f62..a79aed8 100644 --- a/SODA/SodaClient.cs +++ b/SODA/SodaClient.cs @@ -69,7 +69,7 @@ public SodaClient(string host, string appToken, string username, string password /// The user account that this client will use for Authentication during each request. /// The password for the specified that this client will use for Authentication during each request. /// Thrown if no is provided. - public SodaClient(string host, string username, string password) + public SodaClient(string host, string username, string password) : this(host, null, username, password) { } @@ -116,7 +116,7 @@ public IEnumerable GetMetadataPage(int page) { if (page <= 0) throw new ArgumentOutOfRangeException("page", "Resouce metadata catalogs begin on page 1."); - + var catalogUri = SodaUri.ForMetadataList(Host, page); //an entry of raw data contains some, but not all, of the fields required to populate a ResourceMetadata @@ -142,7 +142,7 @@ public Resource GetResource(string resourceId) where TRow : class { if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - + return new Resource(resourceId, this); } @@ -478,5 +478,138 @@ internal TResult write(Uri uri, string method, TPayload paylo return request.ParseResponse(); } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// One of upsert, replace, or delete + /// The identifier (4x4) for a resource on the Socrata host to target. + /// The permission level of the dataset, can be one of either "public" or "private". + /// A newly created Revision. + /// Thrown if the specified does not match the Socrata 4x4 pattern. + public Revision CreateRevision(string method, string resourceId, string permission = "private") + { + if (FourByFour.IsNotValid(resourceId)) + throw new ArgumentOutOfRangeException("The provided resourceId is not a valid Socrata (4x4) resource identifier.", nameof(resourceId)); + + var revisionUri = SodaUri.ForRevision(Host, resourceId); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); + action["type"] = method; + action["permission"] = permission; + payload["action"] = action; + + var request = new SodaRequest(revisionUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + + Result result = null; + try + { + result = request.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new Revision(result); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// A string of serialized data. + /// The revision created as part of a create revision step. + /// The format of the data. + /// The filename that should be associated with this upload. + /// A indicating success or failure. + /// Thrown if this SodaDSMAPIClient was initialized without authentication credentials. + public Source CreateSource(string data, Revision revision, SodaDataFormat dataFormat = SodaDataFormat.CSV, string filename = "NewUpload") + { + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + var uri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); + + var revisionNumber = revision.GetRevisionNumber(); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject(); + source_type["type"] = "upload"; + source_type["filename"] = filename; + payload["source_type"] = source_type; + + var createSourceRequest = new SodaRequest(uri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + Source source = null; + try + { + source = createSourceRequest.ParseResponse(); + string uploadDataPath = source.Links["bytes"]; + uri = SodaUri.ForUpload(Host, uploadDataPath); + + var fileUploadRequest = new SodaRequest(uri, "POST", null, Username, password, dataFormat, data); + source = fileUploadRequest.ParseResponse(); + + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + source = new Source() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + source = new Source() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return source; + } + + + /// + /// Create the InputSchema from the source. + /// + /// The result of the Source creation + /// A SchemaTransforms object + public SchemaTransforms CreateInputSchema(Source source) + { + return new SchemaTransforms(source); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// A string of serialized data. + /// A string of serialized data. + /// A indicating success or failure. + public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) + { + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); + + var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); + Console.WriteLine(uri); + var applyRequest = new SodaRequest(uri, "PUT", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + Result result = null; + try + { + result = applyRequest.ParseResponse(); + + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new PipelineJob(revision.getRevisionLink(), Username, password, revision.GetRevisionNumber()); + } } } diff --git a/SODA/SodaDSMAPIClient.cs b/SODA/SodaDSMAPIClient.cs deleted file mode 100644 index aeafbaf..0000000 --- a/SODA/SodaDSMAPIClient.cs +++ /dev/null @@ -1,238 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using SODA.Utilities; - -namespace SODA -{ - /// - /// A class for interacting with Socrata Data Portals using the Socrata Dataset Management API. - /// - public class SodaDSMAPIClient - { - /// - /// The url to the Socrata Open Data Portal this client targets. - /// - public readonly string Host; - - /// - /// The user account that this client uses for Authentication during each request. - /// - /// - /// Authentication is only necessary when accessing datasets that have been marked as private or when making write requests (PUT, POST, and DELETE). - /// See http://dev.socrata.com/docs/authentication.html for more information. - /// - public readonly string Username; - - //not publicly readable, can only be set in a constructor - private readonly string password; - - /// - /// If set, the number of milliseconds to wait before requests to the timeout and throw a . - /// If unset, the default value is that of . - /// - public int? RequestTimeout { get; set; } - - /// - /// Initialize a new SodaClient for the specified Socrata host, using the specified application token and Authentication credentials. - /// - /// The Socrata Open Data Portal that this client will target. - /// The Socrata application token that this client will use for all requests. - /// The user account that this client will use for Authentication during each request. - /// The password for the specified that this client will use for Authentication during each request. - /// Thrown if no is provided. - public SodaDSMAPIClient(string host, string appToken, string username, string password) - { - if (String.IsNullOrEmpty(host)) - throw new ArgumentException("host", "A host is required"); - - Host = SodaUri.enforceHttps(host); - AppToken = appToken; - Username = username; - this.password = password; - } - - /// - /// Initialize a new SodaClient for the specified Socrata host, using the specified Authentication credentials. - /// - /// The Socrata Open Data Portal that this client will target. - /// The user account that this client will use for Authentication during each request. - /// The password for the specified that this client will use for Authentication during each request. - /// Thrown if no is provided. - public SodaDSMAPIClient(string host, string username, string password) - : this(host, null, username, password) - { - } - /// - /// Replace any existing rows with the payload data, using the specified resource identifier. - /// - /// One of upsert, replace, or delete - /// The identifier (4x4) for a resource on the Socrata host to target. - /// The permission level of the dataset, can be one of either "public" or "private". - /// A newly created Revision. - /// Thrown if the specified does not match the Socrata 4x4 pattern. - public Revision CreateRevision(string method, string resourceId, string permission = "private") - { - if (FourByFour.IsNotValid(resourceId)) - throw new ArgumentOutOfRangeException("The provided resourceId is not a valid Socrata (4x4) resource identifier.", nameof(resourceId)); - - revisionUri = SodaUri.ForRevision(Host, resourceId); - var dataFormat = DataFormat.JSON; - - // Construct Revision Request body - Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); - Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); - action["type"] = method; - action["permission"] = permission; - payload["action"] = action; - - logger.Info(revisionUri); - Console.WriteLine(revisionUri); - var request = new SodaRequest(revisionUri, "POST", Username, password, dataFormat, payload.ToString()); - - Result result = null; - try - { - result = request.ParseResponse(); - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; - } - catch (Exception ex) - { - result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; - } - return new Revision(result); - } - - /// - /// Replace any existing rows with the payload data, using the specified resource identifier. - /// - /// A string of serialized data. - /// The revision created as part of a create revision step. - /// The format of the data. - /// The filename that should be associated with this upload. - /// A indicating success or failure. - /// Thrown if this SodaDSMAPIClient was initialized without authentication credentials. - public Source CreateSource(string data, Revision revision, DataFormat dataFormat, string filename = "NewUpload") - { - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) - throw new InvalidOperationException("Write operations require an authenticated client."); - - var uri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); - - revisionNumber = revision.GetRevisionNumber(); - - // Construct Revision Request body - Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); - Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject(); - source_type["type"] = "upload"; - source_type["filename"] = filename; - payload["source_type"] = source_type; - - logger.Info(uri); - Console.WriteLine(uri); - var createSourceRequest = new SodaRequest(uri, "POST", Username, password, DataFormat.JSON, payload.ToString()); - Source source = null; - try - { - source = createSourceRequest.ParseResponse(); - string uploadDataPath = source.Links["bytes"]; - uri = SodaUri.ForUpload(Host, uploadDataPath); - Console.WriteLine(uri); - - var fileUploadRequest = new SodaRequest(uri, "POST", Username, password, dataFormat, data); - source = fileUploadRequest.ParseResponse(); - - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - source = new Source() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; - } - catch (Exception ex) - { - source = new Source() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; - } - return source; - } - - - /// - /// Create the InputSchema from the source. - /// - /// The result of the Source creation - /// A SchemaTransforms object - public SchemaTransforms CreateInputSchema(Source source) - { - return new SchemaTransforms(source); - } - - /// - /// Replace any existing rows with the payload data, using the specified resource identifier. - /// - /// A string of serialized data. - /// A indicating success or failure. - public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) - { - Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); - payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); - - var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); - logger.Info(uri); - Console.WriteLine(uri); - var applyRequest = new SodaRequest(uri, "PUT", Username, password, DataFormat.JSON, payload.ToString()); - Result result = null; - try - { - result = applyRequest.ParseResponse(); - - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; - } - catch (Exception ex) - { - result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; - } - return new PipelineJob(revisionUri, Username, password, revisionNumber); - } - - /// - /// Replace any existing rows with the payload data, using the specified resource identifier. - /// - /// A string of serialized data. - /// A string of serialized data. - /// A indicating success or failure. - public PipelineJob Apply(SchemaTransforms inputSchema, Revision revision) - { - Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); - payload["output_schema_id"] = inputSchema.GetOutputSchemaId(); - - var uri = SodaUri.ForApply(Host, revision.GetApplyEndpoint()); - logger.Info(uri); - Console.WriteLine(uri); - var applyRequest = new SodaRequest(uri, "PUT", Username, password, DataFormat.JSON, payload.ToString()); - Result result = null; - try - { - result = applyRequest.ParseResponse(); - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; - } - catch (Exception ex) - { - result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; - } - return new PipelineJob(revisionUri, Username, password, revisionNumber); - } - } -} diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index e5d0c9e..f7017ba 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -222,6 +222,7 @@ public static Uri ForCategoryPage(string socrataHost, string category) return new Uri(url); } + /// /// Create a revision Uri the specified resource on the specified Socrata host. /// @@ -230,7 +231,13 @@ public static Uri ForCategoryPage(string socrataHost, string category) /// A revision Uri for the specified resource on the specified Socrata host. public static Uri ForRevision(string socrataHost, string resourceId) { - string url = String.Format("{0}/api/publishing/v1/revision/{1}", socrataHost, resourceId); + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (FourByFour.IsNotValid(resourceId)) + throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}/api/publishing/v1/revision/{1}", enforceHttps(socrataHost), resourceId); return new Uri(url); } @@ -238,25 +245,37 @@ public static Uri ForRevision(string socrataHost, string resourceId) /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. /// /// The Socrata host to target. - /// The identifier (4x4) for a resource on the Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. /// A query Uri for the specified resource on the specified Socrata host. - public static Uri ForSource(string socrataHost, string sourceEndpoint) + public static Uri ForUpload(string socrataHost, string uploadEndpoint) { - string url = String.Format("{0}{1}", socrataHost, sourceEndpoint.Replace("\"", "")); + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(uploadEndpoint)) + throw new ArgumentOutOfRangeException("uploadEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), uploadEndpoint.Replace("\"", "")); return new Uri(url); } - /// /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. /// /// The Socrata host to target. - /// The identifier (4x4) for a resource on the Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. /// A query Uri for the specified resource on the specified Socrata host. - public static Uri ForUpload(string socrataHost, string uploadEndpoint) + public static Uri ForSource(string socrataHost, string sourceEndpoint) { - string url = String.Format("{0}{1}", socrataHost, uploadEndpoint.Replace("\"", "")); + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(sourceEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), sourceEndpoint.Replace("\"", "")); return new Uri(url); + } /// @@ -267,7 +286,13 @@ public static Uri ForUpload(string socrataHost, string uploadEndpoint) /// A query Uri for the specified resource on the specified Socrata host. public static Uri ForApply(string socrataHost, string applyEndpoint) { - string url = String.Format("{0}{1}", socrataHost, applyEndpoint.Replace("\"", "")); + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(applyEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), applyEndpoint.Replace("\"", "")); return new Uri(url); } @@ -275,11 +300,14 @@ public static Uri ForApply(string socrataHost, string applyEndpoint) /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. /// /// The Socrata host to target. - /// The identifier (4x4) for a resource on the Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. /// A query Uri for the specified resource on the specified Socrata host. - public static Uri ForJob(Uri revisionEndpoint, long revisionNumber) + public static Uri ForJob(string socrataHost, long revisionNumber) { - string url = String.Format("{0}/{1}/", revisionEndpoint.ToString(), revisionNumber); + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + string url = String.Format("{0}/{1}/", enforceHttps(socrataHost), revisionNumber); return new Uri(url); } } From 3c143f43fbd2601a6e40674eb80c08ad2b4d8e52 Mon Sep 17 00:00:00 2001 From: Moore Date: Thu, 10 Oct 2019 17:54:55 -0500 Subject: [PATCH 3/8] create datasets; create revisions; manage errors --- README.md | 109 +++++++++++++++++++++++---------- SODA/AppliedTransform.cs | 66 ++++++++++++++++++-- SODA/PipelineJob.cs | 42 +++++++------ SODA/Revision.cs | 20 ++++++- SODA/SODA.csproj | 1 + SODA/SchemaTransforms.cs | 11 ++-- SODA/SodaClient.cs | 123 ++++++++++++++++++++++++++++---------- SODA/SodaRequest.cs | 7 ++- SODA/Source.cs | 85 +++++++++++++++++--------- SODA/Utilities/SodaUri.cs | 32 +++++++++- 10 files changed, 370 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 8d159b1..2039e74 100644 --- a/README.md +++ b/README.md @@ -66,44 +66,87 @@ For more details on when to use SODA vs the Socrata Data Management API, see the ```c# using System; using SODA; +using System.Diagnostics; -namespace MyApp +namespace SocrataTest { - class MyETLClass - { - static void Main(string[] args) + class Program { - // Initialize the client - PipelineClient pipelineClient = new SodaDSMAPIClient("https://my.data.socrata.com", "username", "password"); - - // Read in File (or other source) - string filepath = "C:\\Users\\mysource\\outputs\\output.csv"; - string csv = System.IO.File.ReadAllText(filepath); - - // Create a REVISION - Revision revision = pipelineClient.CreateRevision("replace", "1234-abcd"); - - // Upload the file as a new source - Source source = pipelineClient.CreateSource(csv, revision, DataFormat.CSV, filename="MyNewFile") - - // Await data upload - source.AwaitCompletion() - - // Get the schema of the new (latest) source - SchemaTransforms input = pipelineClient.CreateInputSchema(source); - - // Do transforms - // AppliedTransforms output = input.ChangeColumnDisplayName("oldname","newname").ChangeColumnDescription("newname","New description").Run(); - // - - // Apply the revision to replace/update the dataset - PipelineJob job = pipelineClient.Apply(input, revision); - - // Await the completion of the revision and output the processing log - job.AwaitCompletion(); + static void Main(string[] args) + { + // Initialize the client + SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); + + // Read in File (or other source) + string filepath = "C:\\Users\\{user}\\Desktop\\test.csv"; + string csv = System.IO.File.ReadAllText(filepath); + Debug.WriteLine(csv); + + // Create a Dataset - either public or private (default: private) + Revision dataset = pipelineClient.CreateDataset("MyNewDataset", "public"); + + string datasetId = dataset.GetFourFour(); + Console.WriteLine(datasetId); + + Source source = pipelineClient.CreateSource(csv, dataset, SodaDataFormat.CSV, "File"); + SchemaTransforms input = pipelineClient.CreateInputSchema(source); + AppliedTransform output = input.Run(); + output.AwaitCompletion(pipelineClient, status => { }); + + // Check for Errors + if (output.GetErrorCount() > 0) + { + Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); + pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output); + // Optional Throw new Error... + } + + // Apply the revision to the dataset + PipelineJob job = pipelineClient.Apply(output, dataset); + + // Await the completion of the revision and output the processing log + job.AwaitCompletion(status => Console.WriteLine(status)); + + // CREATING A REVISION + // Create a Revision (either update, replace, or delete) + Revision revision = pipelineClient.CreateRevision("update", datasetId); + + // Upload the file as a new source + Source newSource = pipelineClient.CreateSource(csv, revision, SodaDataFormat.CSV, "MyNewFile"); + //Console.WriteLine(source.GetSchemaId()); + // Get the schema of the new (latest) source + SchemaTransforms newInput = pipelineClient.CreateInputSchema(newSource); + + + // Do transforms + // TODO: + // SchemaTransforms output = input.ChangeColumnDisplayName("oldname","newname").ChangeColumnDescription("newname","New description").Run(); + // + + // Run the output transforms + AppliedTransform newOutput = newInput.Run(); + + // Transforms are applied asynchronously, so we need to wait for them to complete + newOutput.AwaitCompletion(pipelineClient, status => { }); + + // Check for Errors + if(output.GetErrorCount() > 0) + { + Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); + pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output); + // Optional Throw new Error... + } + + // Apply the revision to replace/update the dataset + PipelineJob newJob = pipelineClient.Apply(newOutput, revision); + + // Await the completion of the revision and output the processing log + newJob.AwaitCompletion(status => Console.WriteLine(status) ); + + } } - } } + ``` ## Build diff --git a/SODA/AppliedTransform.cs b/SODA/AppliedTransform.cs index 2436bca..696ac01 100644 --- a/SODA/AppliedTransform.cs +++ b/SODA/AppliedTransform.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; -using System.Linq; -using System.Runtime.Serialization; +using System; using SODA.Utilities; namespace SODA @@ -10,14 +9,69 @@ namespace SODA /// public class AppliedTransform { - Dictionary resource; - public AppliedTransform(Dictionary transform) - { + /// + /// Source object. + /// + Source source; + /// + /// Create an applied transform object based off a source. + /// + /// AppliedTransform + public AppliedTransform(Source source) + { + this.source = source; } + + /// + /// Retrieve the Output schema id. + /// + /// Error count public string GetOutputSchemaId() { - return this.resource["schemas"][0]["output_schemas"][0]["id"]; + return this.source.GetSchemaId(); + } + + /// + /// Retrieve Input schema ID. + /// + /// Error count + public string GetInputSchemaId() + { + return this.source.GetInputSchemaId(); + } + + /// + /// Retrieve the error count. + /// + /// Error count + public int GetErrorCount() + { + return this.source.GetErrorCount(); + } + + /// + /// Retrieve the error endpoint. + /// + /// Error endpoint + public string GetErrorRowEndpoint() + { + return this.source.GetErrorRowEndPoint(); + } + + /// + /// Await completion of transforms. + /// + /// The current Soda client + /// Lambda output + public void AwaitCompletion(SodaClient client, Action lambda) + { + this.source = client.GetSource(this.source); + while (!this.source.IsComplete(lambda)) + { + this.source = client.GetSource(this.source); + System.Threading.Thread.Sleep(1000); + } } } } diff --git a/SODA/PipelineJob.cs b/SODA/PipelineJob.cs index 2198a49..995ae51 100644 --- a/SODA/PipelineJob.cs +++ b/SODA/PipelineJob.cs @@ -9,31 +9,37 @@ namespace SODA /// public class PipelineJob { + /// + /// Socrata username. + /// public string Username; + /// + /// Socrata password. + /// private string password; + /// + /// The revision endpoint. + /// public Uri revisionEndpoint { get; set; } - public PipelineJob(string revEndpoint, string user, string pass, long revNum) + /// + /// Apply the source, transforms, and update to the specified dataset. + /// + /// the JobURI. + /// Username. + /// Password. + public PipelineJob(Uri jobUri, string user, string pass) { Username = user; password = pass; - - revisionEndpoint = SodaUri.ForJob(revEndpoint, revNum); - Console.WriteLine(revisionEndpoint); - var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, Username, password, SodaDataFormat.JSON); - Result r = null; - try - { - r = jobRequest.ParseResponse(); - Console.WriteLine(r.Resource["task_sets"][0]["status"]); - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - r = new Result() { Message = webException.Message, IsError = true, ErrorCode = message }; - } + revisionEndpoint = jobUri; } - public void AwaitCompletion() + + /// + /// Await the completion of the update, optionally output the status. + /// + /// A lambda function for outputting status if desired. + public void AwaitCompletion(Action lambda) { string status = ""; Result r = null; @@ -50,7 +56,7 @@ public void AwaitCompletion() r = new Result() { Message = webException.Message, IsError = true, ErrorCode = message }; } status = r.Resource["task_sets"][0]["status"]; - Console.WriteLine(status); + lambda(status); System.Threading.Thread.Sleep(1000); } } diff --git a/SODA/Revision.cs b/SODA/Revision.cs index 72817e5..0569945 100644 --- a/SODA/Revision.cs +++ b/SODA/Revision.cs @@ -25,18 +25,32 @@ public Revision(Result result) } + /// + /// Get the current revision number. + /// public long GetRevisionNumber() { return this.result.Resource["revision_seq"]; } + /// + /// Get the dataset ID of the current revision. + /// + public string GetFourFour() + { + return this.result.Resource["fourfour"]; + } + + /// + /// Get the revision endpoint. + /// public string getRevisionLink() { - return this.result.Links["self"]; + return this.result.Links["show"]; } /// - /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// Get the create source link endpoint. /// public string GetSourceEndpoint() { @@ -44,7 +58,7 @@ public string GetSourceEndpoint() } /// - /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// Get the apply link endpoint /// public string GetApplyEndpoint() { diff --git a/SODA/SODA.csproj b/SODA/SODA.csproj index 78d1060..28ae8ed 100644 --- a/SODA/SODA.csproj +++ b/SODA/SODA.csproj @@ -8,6 +8,7 @@ true CSM.SodaDotNet API OpenData Socrata SODA + true diff --git a/SODA/SchemaTransforms.cs b/SODA/SchemaTransforms.cs index 20f2ddb..5083b1c 100644 --- a/SODA/SchemaTransforms.cs +++ b/SODA/SchemaTransforms.cs @@ -10,14 +10,14 @@ public class SchemaTransforms /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// - Dictionary source; + Source source; /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// public SchemaTransforms(Source source) { - this.source = source.Resource; + this.source = source; } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. @@ -57,7 +57,7 @@ public void AddColumn() /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// - public Dictionary GetSource() + public Source GetSource() { return this.source; } @@ -66,9 +66,12 @@ public Dictionary GetSource() /// public string GetOutputSchemaId() { - return this.source["schemas"][0]["output_schemas"][0]["id"]; + return this.source.GetSchemaId(); } + /// + /// Apply the transforms; Currently a NO-OP. + /// public AppliedTransform Run() { return new AppliedTransform(this.source); diff --git a/SODA/SodaClient.cs b/SODA/SodaClient.cs index a79aed8..502f0b7 100644 --- a/SODA/SodaClient.cs +++ b/SODA/SodaClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net; using SODA.Utilities; @@ -479,10 +480,50 @@ internal TResult write(Uri uri, string method, TPayload paylo return request.ParseResponse(); } + /// + /// Create a new dataset with a given name and permission level. + /// + /// The dataset name + /// The permission level of the dataset, can be one of either "public" or "private". + /// A newly created Revision. + public Revision CreateDataset(string name, string permission = "private") + { + var revisionUri = SodaUri.ForRevision(Host); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject metadata = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); + metadata["name"] = name; + action["type"] = "replace"; + action["permission"] = permission; + payload["action"] = action; + payload["metadata"] = metadata; + + var request = new SodaRequest(revisionUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + + Result result = null; + try + { + result = request.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new Revision(result); + } + + /// /// Replace any existing rows with the payload data, using the specified resource identifier. /// - /// One of upsert, replace, or delete + /// One of update, replace, or delete /// The identifier (4x4) for a resource on the Socrata host to target. /// The permission level of the dataset, can be one of either "public" or "private". /// A newly created Revision. @@ -521,7 +562,7 @@ public Revision CreateRevision(string method, string resourceId, string permissi } /// - /// Replace any existing rows with the payload data, using the specified resource identifier. + /// Creates the source for the specified revision. /// /// A string of serialized data. /// The revision created as part of a create revision step. @@ -534,41 +575,43 @@ public Source CreateSource(string data, Revision revision, SodaDataFormat dataFo if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); - var uri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); - - var revisionNumber = revision.GetRevisionNumber(); + var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); + Console.WriteLine(sourceUri.ToString()); + revision.GetRevisionNumber(); // Construct Revision Request body Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject parse_option = new Newtonsoft.Json.Linq.JObject(); source_type["type"] = "upload"; source_type["filename"] = filename; + parse_option["parse_source"] = true; payload["source_type"] = source_type; - - var createSourceRequest = new SodaRequest(uri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); - Source source = null; - try - { - source = createSourceRequest.ParseResponse(); - string uploadDataPath = source.Links["bytes"]; - uri = SodaUri.ForUpload(Host, uploadDataPath); - - var fileUploadRequest = new SodaRequest(uri, "POST", null, Username, password, dataFormat, data); - source = fileUploadRequest.ParseResponse(); - - } - catch (WebException webException) - { - string message = webException.UnwrapExceptionMessage(); - source = new Source() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; - } - catch (Exception ex) - { - source = new Source() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; - } - return source; + payload["parse_options"] = parse_option; + + var createSourceRequest = new SodaRequest(sourceUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + Result sourceOutput = createSourceRequest.ParseResponse(); + string uploadDataPath = sourceOutput.Links["bytes"]; + var uploadUri = SodaUri.ForUpload(Host, uploadDataPath); + Console.WriteLine(uploadUri.ToString()); + var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, Username, password, dataFormat, data); + fileUploadRequest.SetDataType(SodaDataFormat.JSON); + Result result = fileUploadRequest.ParseResponse(); + return new Source(result); } + /// + /// Get the specified source data. + /// + /// The result of the Source creation + /// A The updated Source object + public Source GetSource(Source source) + { + var sourceUri = SodaUri.ForSource(Host, source.Self()); + var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, Username, password, SodaDataFormat.JSON, ""); + Result result = sourceUpdateResponse.ParseResponse(); + return new Source(result); + } /// /// Create the InputSchema from the source. @@ -581,18 +624,33 @@ public SchemaTransforms CreateInputSchema(Source source) } /// - /// Replace any existing rows with the payload data, using the specified resource identifier. + /// Export the error rows (if present). + /// + /// The output file (csv) + /// The specified transformed output + public void ExportErrorRows(string filepath, AppliedTransform output) + { + var endpoint = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId()); + var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint); + Console.WriteLine(errorRowsUri.ToString()); + var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, Username, password, SodaDataFormat.CSV, ""); + var result = downloadRowsRequest.ParseResponse(); + System.IO.File.WriteAllText(filepath, result); + } + + /// + /// Apply the source, transforms, and update to the specified dataset. /// /// A string of serialized data. /// A string of serialized data. - /// A indicating success or failure. + /// A for determining success of failure. public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) { Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); - Console.WriteLine(uri); + Console.WriteLine(uri.ToString()); var applyRequest = new SodaRequest(uri, "PUT", null, Username, password, SodaDataFormat.JSON, payload.ToString()); Result result = null; try @@ -609,7 +667,8 @@ public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) { result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; } - return new PipelineJob(revision.getRevisionLink(), Username, password, revision.GetRevisionNumber()); + + return new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), Username, password); } } } diff --git a/SODA/SodaRequest.cs b/SODA/SodaRequest.cs index db482cb..c9c96c1 100644 --- a/SODA/SodaRequest.cs +++ b/SODA/SodaRequest.cs @@ -75,6 +75,7 @@ internal SodaRequest(Uri uri, string method, string appToken, string username, s break; case SodaDataFormat.CSV: this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("text/csv")); + this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); break; case SodaDataFormat.XML: this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/rdf+xml")); @@ -97,7 +98,11 @@ internal SodaRequest(Uri uri, string method, string appToken, string username, s } } } - + + internal void SetDataType(SodaDataFormat format) + { + dataFormat = format; + } /// /// Send this SodaRequest's webRequest and interpret the response. /// diff --git a/SODA/Source.cs b/SODA/Source.cs index d34b013..a7075e4 100644 --- a/SODA/Source.cs +++ b/SODA/Source.cs @@ -1,57 +1,88 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.Serialization; using SODA.Utilities; +using System.Security.Permissions; namespace SODA + { /// - /// Gets the information related to the Source. + /// A class for accessing the Source object. /// - [DataContract] public class Source { /// - /// Gets the number of errors. + /// The result of a source being created. + /// + Result result; + + /// + /// A class for handling Sources. /// - [DataMember(Name = "Errors")] - public int Errors { get; private set; } + public Source(Result result) + { + this.result = result; + } /// - /// Gets the explanatory text about this result. + /// Get lastest Schema ID. /// - [DataMember(Name = "message")] - public string Message { get; internal set; } + public string GetSchemaId() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["id"]; + } /// - /// Gets a flag indicating if one or more errors occured. + /// Get Input schema ID. /// - [DataMember(Name = "error")] - public bool IsError { get; internal set; } + /// Input Schema ID + public string GetInputSchemaId() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["input_schema_id"]; + } /// - /// Gets data about any errors that occured. + /// Retrieve the error count. /// - [DataMember(Name = "code")] - public string ErrorCode { get; internal set; } + /// Error count + public int GetErrorCount() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["error_count"]; + } /// - /// Gets any additional data associated with this result. + /// Retrieve the error count. /// - [DataMember(Name = "data")] - public dynamic Data { get; internal set; } + /// Error count + public Boolean IsComplete(Action lambda) + { + string completed_at = this.result.Resource["schemas"][0]["output_schemas"][0]["completed_at"]; + if(String.IsNullOrEmpty(completed_at)) + { + lambda("Working..."); + return false; + } else + { + lambda(String.Format("Completed at: {0}", completed_at)); + return true; + } + } /// - /// Get data related to the resource. + /// Error row endpoint. /// - [DataMember(Name = "resource")] - public Dictionary Resource { get; set; } + /// Error row endpoint + public string GetErrorRowEndPoint() + { + return this.result.Links["input_schema_links"]["output_schema_links"]["schema_errors"]; + } /// - /// Gets links provided for gathering additional resources. + /// Get the self link. /// - [DataMember(Name = "links")] - public Dictionary Links { get; set; } + /// Current endpoint + public string Self() + { + return this.result.Links["show"]; + } } -} +} \ No newline at end of file diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index f7017ba..3dd0a34 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -241,6 +241,20 @@ public static Uri ForRevision(string socrataHost, string resourceId) return new Uri(url); } + /// + /// Create a revision Uri the specified resource on the specified Socrata host. + /// + /// The Socrata host to target. + /// A revision Uri for the specified resource on the specified Socrata host. + public static Uri ForRevision(string socrataHost) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + string url = String.Format("{0}/api/publishing/v1/revision", enforceHttps(socrataHost)); + return new Uri(url); + } + /// /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. /// @@ -302,12 +316,26 @@ public static Uri ForApply(string socrataHost, string applyEndpoint) /// The Socrata host to target. /// The identifier (4x4) for a resource on the Socrata host to target. /// A query Uri for the specified resource on the specified Socrata host. - public static Uri ForJob(string socrataHost, long revisionNumber) + public static Uri ForJob(string socrataHost, string revisionEndpoint) { if (String.IsNullOrEmpty(socrataHost)) throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); - string url = String.Format("{0}/{1}/", enforceHttps(socrataHost), revisionNumber); + string url = String.Format("{0}{1}", enforceHttps(socrataHost), revisionEndpoint.Replace("\"", "")); + return new Uri(url); + } + + /// + /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// + /// The Socrata host to target. + /// The error endpoint + /// A error Uri for the specified resource on the specified Socrata host. + public static Uri ForErrorRows(string socrataHost, string errorEndpoint) + { + if (String.IsNullOrEmpty(socrataHost) || String.IsNullOrEmpty(errorEndpoint)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + string url = String.Format("{0}{1}", enforceHttps(socrataHost), errorEndpoint); return new Uri(url); } } From 9ec6274ad7ed74924908d4c1af057dbb72ee0713 Mon Sep 17 00:00:00 2001 From: Moore Date: Thu, 10 Oct 2019 17:58:49 -0500 Subject: [PATCH 4/8] update readme --- README.md | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2039e74..1bd5b30 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,6 @@ namespace SocrataTest // Read in File (or other source) string filepath = "C:\\Users\\{user}\\Desktop\\test.csv"; string csv = System.IO.File.ReadAllText(filepath); - Debug.WriteLine(csv); // Create a Dataset - either public or private (default: private) Revision dataset = pipelineClient.CreateDataset("MyNewDataset", "public"); @@ -106,10 +105,32 @@ namespace SocrataTest // Await the completion of the revision and output the processing log job.AwaitCompletion(status => Console.WriteLine(status)); - + + } + } +} +``` +```cs +using System; +using SODA; +using System.Diagnostics; + +namespace SocrataTest +{ + class Program + { + static void Main(string[] args) + { + // Initialize the client + SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); + + // Read in File (or other source) + string filepath = "C:\\Users\\{user}\\Desktop\\test.csv"; + string csv = System.IO.File.ReadAllText(filepath); + // CREATING A REVISION // Create a Revision (either update, replace, or delete) - Revision revision = pipelineClient.CreateRevision("update", datasetId); + Revision revision = pipelineClient.CreateRevision("update", "1234-abcd"); // Upload the file as a new source Source newSource = pipelineClient.CreateSource(csv, revision, SodaDataFormat.CSV, "MyNewFile"); @@ -117,12 +138,6 @@ namespace SocrataTest // Get the schema of the new (latest) source SchemaTransforms newInput = pipelineClient.CreateInputSchema(newSource); - - // Do transforms - // TODO: - // SchemaTransforms output = input.ChangeColumnDisplayName("oldname","newname").ChangeColumnDescription("newname","New description").Run(); - // - // Run the output transforms AppliedTransform newOutput = newInput.Run(); From 05da52a2e40ff8c11ffca2c87e715db017442691 Mon Sep 17 00:00:00 2001 From: Peter Moore Date: Fri, 11 Oct 2019 14:26:20 -0500 Subject: [PATCH 5/8] adding tests --- SODA.Tests/SodaClientTests.cs | 32 +++++++++++++++++++ SODA.Tests/SodaUriTests.cs | 58 ++++++++++++++++++++++++++++++++++- SODA/SodaClient.cs | 46 ++++++++++++++++++++++++--- SODA/Utilities/SodaUri.cs | 3 ++ 4 files changed, 134 insertions(+), 5 deletions(-) diff --git a/SODA.Tests/SodaClientTests.cs b/SODA.Tests/SodaClientTests.cs index 1f569f9..46d7ae5 100644 --- a/SODA.Tests/SodaClientTests.cs +++ b/SODA.Tests/SodaClientTests.cs @@ -266,5 +266,37 @@ public void DeleteRow_Using_Anonymous_Client_Throws_InvalidOperationException() { Assert.That(() => mockClient.DeleteRow(StringMocks.NonEmptyInput, StringMocks.ResourceId), Throws.TypeOf()); } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaClient")] + public void DSMAPI_With_Invalid_ResourceId_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => mockClient.CreateRevision(StringMocks.NonEmptyInput, input), Throws.TypeOf()); + } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaClient")] + public void DSMAPI_With_NULL_Or_Empty_Values_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => mockClient.CreateDataset(input), Throws.TypeOf()); + Assert.That(() => mockClient.CreateRevision(input, StringMocks.ResourceId), Throws.TypeOf()); + Assert.That(() => mockClient.CreateSource(input, null, SodaDataFormat.CSV), Throws.TypeOf()); + Assert.That(() => mockClient.ExportErrorRows(input, null), Throws.TypeOf()); + + } + + [Category("SodaClient")] + public void DSMAPI_With_NUL_Values_Throws_ArgumentOutOfRangeException() + { + Assert.That(() => mockClient.CreateSource(StringMocks.NonEmptyInput, null, SodaDataFormat.CSV), Throws.TypeOf()); + + Assert.That(() => mockClient.GetSource(null), Throws.TypeOf()); + Assert.That(() => mockClient.CreateInputSchema(null), Throws.TypeOf()); + + Assert.That(() => mockClient.ExportErrorRows(StringMocks.NonEmptyInput, null), Throws.TypeOf()); + Assert.That(() => mockClient.Apply(null, null), Throws.TypeOf()); + } } } \ No newline at end of file diff --git a/SODA.Tests/SodaUriTests.cs b/SODA.Tests/SodaUriTests.cs index d654c38..b4bb8ba 100644 --- a/SODA.Tests/SodaUriTests.cs +++ b/SODA.Tests/SodaUriTests.cs @@ -47,12 +47,20 @@ public void All_Methods_Return_Uri_With_Socrata_Domain_As_Host() uri = SodaUri.ForSource(StringMocks.Host, "/" + StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + uri = null; + uri = SodaUri.ForUpload(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + uri = null; uri = SodaUri.ForApply(StringMocks.Host, "/" + StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); uri = null; - uri = SodaUri.ForJob(StringMocks.Host, 1); + uri = SodaUri.ForJob(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForErrorRows(StringMocks.Host, "/" + StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); } @@ -114,6 +122,30 @@ public void All_Methods_Return_Uri_Using_HTTPS() uri = null; uri = SodaUri.ForCategoryPage("http://" + StringMocks.Host, StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForRevision("http://" + StringMocks.Host, StringMocks.ResourceId); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForSource("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForUpload("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForApply("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForJob("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForErrorRows("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); } [TestCase(StringMocks.NullInput)] @@ -344,6 +376,30 @@ public void ForCategoryPage_With_Empty_Category_Throws_ArgumentException(string Assert.That(() => SodaUri.ForCategoryPage(StringMocks.Host, input), Throws.TypeOf()); } + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaUri")] + public void DSMAPI_With_Invalid_Endpoint_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => SodaUri.ForRevision(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForUpload(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForSource(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForJob(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForApply(StringMocks.Host, input), Throws.TypeOf()); + } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaUri")] + public void DSMAPI_With_Invalid_Host_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => SodaUri.ForRevision(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForUpload(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForSource(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForJob(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForApply(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + } + [Test] [Category("SodaUri")] public void ForCategoryPage_With_Valid_Arguments_Creates_CategoryPage_Uri() diff --git a/SODA/SodaClient.cs b/SODA/SodaClient.cs index 502f0b7..4fcaec4 100644 --- a/SODA/SodaClient.cs +++ b/SODA/SodaClient.cs @@ -488,6 +488,12 @@ internal TResult write(Uri uri, string method, TPayload paylo /// A newly created Revision. public Revision CreateDataset(string name, string permission = "private") { + if (String.IsNullOrEmpty(name)) + throw new ArgumentException("Dataset name required.", "name"); + + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + var revisionUri = SodaUri.ForRevision(Host); // Construct Revision Request body @@ -530,8 +536,15 @@ public Revision CreateDataset(string name, string permission = "private") /// Thrown if the specified does not match the Socrata 4x4 pattern. public Revision CreateRevision(string method, string resourceId, string permission = "private") { + + if (String.IsNullOrEmpty(method)) + throw new ArgumentException("Method must be specified either update, replace, or delete.", "method"); + if (FourByFour.IsNotValid(resourceId)) - throw new ArgumentOutOfRangeException("The provided resourceId is not a valid Socrata (4x4) resource identifier.", nameof(resourceId)); + throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); var revisionUri = SodaUri.ForRevision(Host, resourceId); @@ -572,9 +585,15 @@ public Revision CreateRevision(string method, string resourceId, string permissi /// Thrown if this SodaDSMAPIClient was initialized without authentication credentials. public Source CreateSource(string data, Revision revision, SodaDataFormat dataFormat = SodaDataFormat.CSV, string filename = "NewUpload") { + if (String.IsNullOrEmpty(data)) + throw new ArgumentException("Data must be provided.", "data"); + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); + if (revision == null) + throw new ArgumentException("Revision required.", "revision"); + var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); Console.WriteLine(sourceUri.ToString()); revision.GetRevisionNumber(); @@ -593,7 +612,7 @@ public Source CreateSource(string data, Revision revision, SodaDataFormat dataFo Result sourceOutput = createSourceRequest.ParseResponse(); string uploadDataPath = sourceOutput.Links["bytes"]; var uploadUri = SodaUri.ForUpload(Host, uploadDataPath); - Console.WriteLine(uploadUri.ToString()); + Debug.WriteLine(uploadUri.ToString()); var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, Username, password, dataFormat, data); fileUploadRequest.SetDataType(SodaDataFormat.JSON); Result result = fileUploadRequest.ParseResponse(); @@ -607,6 +626,8 @@ public Source CreateSource(string data, Revision revision, SodaDataFormat dataFo /// A The updated Source object public Source GetSource(Source source) { + if (source == null) + throw new ArgumentException("Source required.", "source"); var sourceUri = SodaUri.ForSource(Host, source.Self()); var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, Username, password, SodaDataFormat.JSON, ""); Result result = sourceUpdateResponse.ParseResponse(); @@ -620,6 +641,8 @@ public Source GetSource(Source source) /// A SchemaTransforms object public SchemaTransforms CreateInputSchema(Source source) { + if (source == null) + throw new ArgumentException("Source required.", "source"); return new SchemaTransforms(source); } @@ -630,9 +653,18 @@ public SchemaTransforms CreateInputSchema(Source source) /// The specified transformed output public void ExportErrorRows(string filepath, AppliedTransform output) { + if (String.IsNullOrEmpty(filepath)) + throw new ArgumentException("Filepath must be specified.", "filepath"); + + if (output == null) + throw new ArgumentException("Applied Transform required.", "output"); + + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + var endpoint = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId()); var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint); - Console.WriteLine(errorRowsUri.ToString()); + Debug.WriteLine(errorRowsUri.ToString()); var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, Username, password, SodaDataFormat.CSV, ""); var result = downloadRowsRequest.ParseResponse(); System.IO.File.WriteAllText(filepath, result); @@ -646,11 +678,17 @@ public void ExportErrorRows(string filepath, AppliedTransform output) /// A for determining success of failure. public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) { + if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + if (outputSchema == null || revision == null) + throw new InvalidOperationException("Both the output schema and revision are required."); + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); - Console.WriteLine(uri.ToString()); + Debug.WriteLine(uri.ToString()); var applyRequest = new SodaRequest(uri, "PUT", null, Username, password, SodaDataFormat.JSON, payload.ToString()); Result result = null; try diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index 3dd0a34..f4a7e87 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -321,6 +321,9 @@ public static Uri ForJob(string socrataHost, string revisionEndpoint) if (String.IsNullOrEmpty(socrataHost)) throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + if (String.IsNullOrEmpty(revisionEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + string url = String.Format("{0}{1}", enforceHttps(socrataHost), revisionEndpoint.Replace("\"", "")); return new Uri(url); } From 4990b170323425e91bd4f435a569eda70d703d9a Mon Sep 17 00:00:00 2001 From: Peter Moore Date: Fri, 11 Oct 2019 14:30:36 -0500 Subject: [PATCH 6/8] update readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bd5b30..c3bb1a7 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,11 @@ IEnumerable payload = GetPayloadData(); client.Upsert(payload, "1234-wxyz"); ``` -**SodaDSMAPIClient** is used for performing Dataset Management API requests +**SodaClient** cam also be used for performing Dataset Management API requests For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#) +Creating datasets: ```c# using System; using SODA; @@ -110,6 +111,8 @@ namespace SocrataTest } } ``` + +Creating update, replace, or delete revisions: ```cs using System; using SODA; From 61e20c3cdd10fb9d97c32688db403e9cce47f2f9 Mon Sep 17 00:00:00 2001 From: peter Moore Date: Tue, 15 Oct 2019 18:35:26 -0700 Subject: [PATCH 7/8] fixes from PR comments --- README.md | 14 ++++----- SODA/PipelineJob.cs | 10 +++---- SODA/SchemaTransforms.cs | 10 +++---- SODA/SodaClient.cs | 61 +++++++++++++++++++-------------------- SODA/Utilities/SodaUri.cs | 10 +++---- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index c3bb1a7..f90d271 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ namespace SocrataTest SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); // Read in File (or other source) - string filepath = "C:\\Users\\{user}\\Desktop\\test.csv"; + string filepath = @"C:\Users\{user}\Desktop\test.csv"; string csv = System.IO.File.ReadAllText(filepath); // Create a Dataset - either public or private (default: private) @@ -97,7 +97,7 @@ namespace SocrataTest if (output.GetErrorCount() > 0) { Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); - pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output); + pipelineClient.ExportErrorRows(@"C:\Users\{user}\Desktop\errors.csv", output); // Optional Throw new Error... } @@ -106,7 +106,7 @@ namespace SocrataTest // Await the completion of the revision and output the processing log job.AwaitCompletion(status => Console.WriteLine(status)); - + } } } @@ -128,9 +128,9 @@ namespace SocrataTest SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); // Read in File (or other source) - string filepath = "C:\\Users\\{user}\\Desktop\\test.csv"; + string filepath = @"C:\Users\{user}\Desktop\test.csv"; string csv = System.IO.File.ReadAllText(filepath); - + // CREATING A REVISION // Create a Revision (either update, replace, or delete) Revision revision = pipelineClient.CreateRevision("update", "1234-abcd"); @@ -151,7 +151,7 @@ namespace SocrataTest if(output.GetErrorCount() > 0) { Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); - pipelineClient.ExportErrorRows("C:\\Users\\{user}\\Desktop\\errors.csv", output); + pipelineClient.ExportErrorRows(@"C:\Users\{user}\Desktop\errors.csv", output); // Optional Throw new Error... } @@ -160,7 +160,7 @@ namespace SocrataTest // Await the completion of the revision and output the processing log newJob.AwaitCompletion(status => Console.WriteLine(status) ); - + } } } diff --git a/SODA/PipelineJob.cs b/SODA/PipelineJob.cs index 995ae51..0894c22 100644 --- a/SODA/PipelineJob.cs +++ b/SODA/PipelineJob.cs @@ -12,7 +12,7 @@ public class PipelineJob /// /// Socrata username. /// - public string Username; + public string username; /// /// Socrata password. /// @@ -23,14 +23,14 @@ public class PipelineJob public Uri revisionEndpoint { get; set; } /// - /// Apply the source, transforms, and update to the specified dataset. + /// Create the pipeline job object. /// /// the JobURI. - /// Username. + /// username. /// Password. public PipelineJob(Uri jobUri, string user, string pass) { - Username = user; + username = user; password = pass; revisionEndpoint = jobUri; } @@ -45,7 +45,7 @@ public void AwaitCompletion(Action lambda) Result r = null; while(status != "successful" && status != "failure") { - var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, Username, password, SodaDataFormat.JSON); + var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, username, password, SodaDataFormat.JSON); try { r = jobRequest.ParseResponse(); diff --git a/SODA/SchemaTransforms.cs b/SODA/SchemaTransforms.cs index 5083b1c..fac591f 100644 --- a/SODA/SchemaTransforms.cs +++ b/SODA/SchemaTransforms.cs @@ -24,35 +24,35 @@ public SchemaTransforms(Source source) /// public void ChangeColumnFieldname() { - + // TODO: WIP } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// public void ChangeColumnDescription() { - + // TODO: WIP } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// public void ChangeColumnDisplayName() { - + // TODO: WIP } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// public void ChangeColumnTransform() { - + // TODO: WIP } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. /// public void AddColumn() { - + // TODO: WIP } /// /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. diff --git a/SODA/SodaClient.cs b/SODA/SodaClient.cs index 4fcaec4..c2df476 100644 --- a/SODA/SodaClient.cs +++ b/SODA/SodaClient.cs @@ -33,7 +33,7 @@ public class SodaClient /// Authentication is only necessary when accessing datasets that have been marked as private or when making write requests (PUT, POST, and DELETE). /// See http://dev.socrata.com/docs/authentication.html for more information. /// - public readonly string Username; + public readonly string username; //not publicly readable, can only be set in a constructor private readonly string password; @@ -59,7 +59,7 @@ public SodaClient(string host, string appToken, string username, string password Host = SodaUri.enforceHttps(host); AppToken = appToken; - Username = username; + username = username; this.password = password; } @@ -210,12 +210,12 @@ public SodaResult Upsert(string payload, SodaDataFormat dataFormat, string resou if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId); - var request = new SodaRequest(uri, "POST", AppToken, Username, password, dataFormat, payload); + var request = new SodaRequest(uri, "POST", AppToken, username, password, dataFormat, payload); SodaResult result = null; try @@ -256,7 +256,7 @@ public SodaResult Upsert(IEnumerable payload, string resourceId) whe if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); string json = Newtonsoft.Json.JsonConvert.SerializeObject(payload); @@ -279,7 +279,7 @@ public IEnumerable BatchUpsert(IEnumerable payload, int if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); Queue queue = new Queue(payload); @@ -337,7 +337,7 @@ public IEnumerable BatchUpsert(IEnumerable payload, int if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); //we create a no-op function that returns false for all inputs @@ -366,12 +366,12 @@ public SodaResult Replace(string payload, SodaDataFormat dataFormat, string reso if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId); - var request = new SodaRequest(uri, "PUT", AppToken, Username, password, dataFormat, payload); + var request = new SodaRequest(uri, "PUT", AppToken, username, password, dataFormat, payload); SodaResult result = null; try @@ -412,7 +412,7 @@ public SodaResult Replace(IEnumerable payload, string resourceId) wh if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); string json = Newtonsoft.Json.JsonConvert.SerializeObject(payload); @@ -437,12 +437,12 @@ public SodaResult DeleteRow(string rowId, string resourceId) if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId, rowId); - var request = new SodaRequest(uri, "DELETE", AppToken, Username, password); + var request = new SodaRequest(uri, "DELETE", AppToken, username, password); return request.ParseResponse(); } @@ -457,7 +457,7 @@ public SodaResult DeleteRow(string rowId, string resourceId) internal TResult read(Uri uri, SodaDataFormat dataFormat = SodaDataFormat.JSON) where TResult : class { - var request = new SodaRequest(uri, "GET", AppToken, Username, password, dataFormat, null, RequestTimeout); + var request = new SodaRequest(uri, "GET", AppToken, username, password, dataFormat, null, RequestTimeout); return request.ParseResponse(); } @@ -475,7 +475,7 @@ internal TResult write(Uri uri, string method, TPayload paylo where TPayload : class where TResult : class { - var request = new SodaRequest(uri, method, AppToken, Username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout); + var request = new SodaRequest(uri, method, AppToken, username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout); return request.ParseResponse(); } @@ -491,7 +491,7 @@ public Revision CreateDataset(string name, string permission = "private") if (String.IsNullOrEmpty(name)) throw new ArgumentException("Dataset name required.", "name"); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var revisionUri = SodaUri.ForRevision(Host); @@ -506,7 +506,7 @@ public Revision CreateDataset(string name, string permission = "private") payload["action"] = action; payload["metadata"] = metadata; - var request = new SodaRequest(revisionUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); Result result = null; try @@ -525,7 +525,6 @@ public Revision CreateDataset(string name, string permission = "private") return new Revision(result); } - /// /// Replace any existing rows with the payload data, using the specified resource identifier. /// @@ -543,10 +542,10 @@ public Revision CreateRevision(string method, string resourceId, string permissi if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); - var revisionUri = SodaUri.ForRevision(Host, resourceId); + var revisionUri = SodaUri.ForRevision(Host, resourceId); // Construct Revision Request body Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); @@ -555,7 +554,7 @@ public Revision CreateRevision(string method, string resourceId, string permissi action["permission"] = permission; payload["action"] = action; - var request = new SodaRequest(revisionUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); Result result = null; try @@ -588,14 +587,14 @@ public Source CreateSource(string data, Revision revision, SodaDataFormat dataFo if (String.IsNullOrEmpty(data)) throw new ArgumentException("Data must be provided.", "data"); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); if (revision == null) throw new ArgumentException("Revision required.", "revision"); var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); - Console.WriteLine(sourceUri.ToString()); + Debug.WriteLine(sourceUri.ToString()); revision.GetRevisionNumber(); // Construct Revision Request body @@ -608,12 +607,12 @@ public Source CreateSource(string data, Revision revision, SodaDataFormat dataFo payload["source_type"] = source_type; payload["parse_options"] = parse_option; - var createSourceRequest = new SodaRequest(sourceUri, "POST", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + var createSourceRequest = new SodaRequest(sourceUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); Result sourceOutput = createSourceRequest.ParseResponse(); string uploadDataPath = sourceOutput.Links["bytes"]; var uploadUri = SodaUri.ForUpload(Host, uploadDataPath); Debug.WriteLine(uploadUri.ToString()); - var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, Username, password, dataFormat, data); + var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, username, password, dataFormat, data); fileUploadRequest.SetDataType(SodaDataFormat.JSON); Result result = fileUploadRequest.ParseResponse(); return new Source(result); @@ -629,7 +628,7 @@ public Source GetSource(Source source) if (source == null) throw new ArgumentException("Source required.", "source"); var sourceUri = SodaUri.ForSource(Host, source.Self()); - var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, Username, password, SodaDataFormat.JSON, ""); + var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, username, password, SodaDataFormat.JSON, ""); Result result = sourceUpdateResponse.ParseResponse(); return new Source(result); } @@ -659,13 +658,13 @@ public void ExportErrorRows(string filepath, AppliedTransform output) if (output == null) throw new ArgumentException("Applied Transform required.", "output"); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var endpoint = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId()); var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint); Debug.WriteLine(errorRowsUri.ToString()); - var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, Username, password, SodaDataFormat.CSV, ""); + var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, username, password, SodaDataFormat.CSV, ""); var result = downloadRowsRequest.ParseResponse(); System.IO.File.WriteAllText(filepath, result); } @@ -675,10 +674,10 @@ public void ExportErrorRows(string filepath, AppliedTransform output) /// /// A string of serialized data. /// A string of serialized data. - /// A for determining success of failure. + /// A for determining success or failure. public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) { - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); if (outputSchema == null || revision == null) @@ -689,7 +688,7 @@ public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); Debug.WriteLine(uri.ToString()); - var applyRequest = new SodaRequest(uri, "PUT", null, Username, password, SodaDataFormat.JSON, payload.ToString()); + var applyRequest = new SodaRequest(uri, "PUT", null, username, password, SodaDataFormat.JSON, payload.ToString()); Result result = null; try { @@ -706,7 +705,7 @@ public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; } - return new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), Username, password); + return new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), username, password); } } } diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index f4a7e87..037f053 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -256,7 +256,7 @@ public static Uri ForRevision(string socrataHost) } /// - /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// Create a URI for uploading the source data. /// /// The Socrata host to target. /// The identifier (4x4) for a resource on the Socrata host to target. @@ -274,7 +274,7 @@ public static Uri ForUpload(string socrataHost, string uploadEndpoint) } /// - /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// URI for the particular source. /// /// The Socrata host to target. /// The identifier (4x4) for a resource on the Socrata host to target. @@ -293,7 +293,7 @@ public static Uri ForSource(string socrataHost, string sourceEndpoint) } /// - /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// URI for Applying the transforms. /// /// The Socrata host to target. /// The identifier (4x4) for a resource on the Socrata host to target. @@ -311,7 +311,7 @@ public static Uri ForApply(string socrataHost, string applyEndpoint) } /// - /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// Create a URI for the Job. /// /// The Socrata host to target. /// The identifier (4x4) for a resource on the Socrata host to target. @@ -329,7 +329,7 @@ public static Uri ForJob(string socrataHost, string revisionEndpoint) } /// - /// Create a Uri for querying the specified resource on the specified Socrata host, using the specified SoqlQuery object. + /// Create a URI for exporting the error rows. /// /// The Socrata host to target. /// The error endpoint From 614cac96c47c8f3a124f35ef36c6c1d1e2f3858d Mon Sep 17 00:00:00 2001 From: peter Moore Date: Wed, 11 Mar 2020 20:44:37 -0700 Subject: [PATCH 8/8] Merge https://github.com/CityofSantaMonica/SODA.NET --- .gitignore | 1 + DEPLOYMENT.md | 8 +- Directory.build.props | 6 +- .../NetCore30.SODA.Tests.csproj | 23 +++++ .../NetCore30.Utilities.Tests.csproj | 27 ++++++ README.md | 43 ++++----- ReleaseNotes.md | 84 ++++++++++-------- SODA.sln | 14 +++ SODA/Models/LocationColumn.cs | 1 + SODA/Models/PhoneColumn.cs | 1 + SODA/Properties/AssemblyInfo.cs | 1 + SODA/Resource.cs | 3 +- SODA/ResourceMetadata.cs | 1 + SODA/SodaRequest.cs | 1 + Utilities/DataFileExporter.cs | 3 - Utilities/Properties/AssemblyInfo.cs | 3 +- appveyor.yml | 7 +- icon.png | Bin 0 -> 3452 bytes 18 files changed, 155 insertions(+), 72 deletions(-) create mode 100644 NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj create mode 100644 NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj create mode 100644 icon.png diff --git a/.gitignore b/.gitignore index a055142..92d8441 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ Desktop.ini *.suo *.user .vs/ +.vscode/ # Build results bin/[Dd]ebug/ diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 8e20101..4afc6c4 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,6 +1,6 @@ # Deploying a new release -Deployment is hard. There are too many manual steps. Let's at least enumerate +Deployment is hard. There are too many manual steps. Let's at least enumerate them so everyone is on the same page. Assuming all changes for the new release are commited, tests have been run and @@ -9,11 +9,11 @@ are passing, and you're happy with the state of things... 1. Create a branch named `release` 1. Update [`ReleaseNotes.md`](ReleaseNotes.md), following the existing format 1. Bump the version number in [`Directory.build.props`](Directory.build.props) and [`appveyor.yml`](appveyor.yml) -1. Push the branch to GitHub and create a pull request +1. Create the NuGet packages (in the local solution root) for the new version: `dotnet pack --output .` +1. Test the NuGet packages! [How to install NuGet package locally](http://stackoverflow.com/questions/10240029/how-to-install-a-nuget-package-nupkg-file-locally) +1. Push the `release` branch to GitHub and create a pull request 1. If the build succeeds, accept the pull request 1. Create and push a tag from the new HEAD `git tag v#.#.#` and `git push --tags` -1. Create the NuGet packages (in the local solution root) for the new version: `dotnet pack --output ..\` -1. Test the NuGet packages! [How to install NuGet package locally](http://stackoverflow.com/questions/10240029/how-to-install-a-nuget-package-nupkg-file-locally) 1. [Create a new release](https://help.github.com/articles/creating-releases) using the tag you just created and pasting in the release notes you just wrote up. Attach a copy of the latest `.nupkg` files generated above. 1. Push the new packages up to NuGet `dotnet nuget push` diff --git a/Directory.build.props b/Directory.build.props index bb9a026..431917d 100644 --- a/Directory.build.props +++ b/Directory.build.props @@ -6,6 +6,10 @@ SODA https://github.com/CityofSantaMonica/SODA.NET MIT - 0.9.0 + icon.png + 0.10.1 + + + \ No newline at end of file diff --git a/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj b/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj new file mode 100644 index 0000000..c8a18c2 --- /dev/null +++ b/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj @@ -0,0 +1,23 @@ + + + + netcoreapp3.0 + + false + + NetCore30.SODA.Tests + + + + + + + + + + + + + + + diff --git a/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj b/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj new file mode 100644 index 0000000..6af4b68 --- /dev/null +++ b/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp3.0 + + false + + NetCore30.SODA.Utilities.Tests + + NetCore30.SODA.Utilities.Tests + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index f90d271..110fde1 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # SODA.NET [![Build status](https://ci.appveyor.com/api/projects/status/yub6lyl79573lufv/branch/master?svg=true)](https://ci.appveyor.com/project/thekaveman/soda-net/branch/master) -A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting -.NET 4.5 and above. +A [Socrata Open Data API](https://dev.socrata.com) (SODA) client library targeting .NET 4.5 and above. ## Getting Started SODA.NET is available as a [NuGet package](https://www.nuget.org/packages/CSM.SodaDotNet/). - Install-Package CSM.SodaDotNet +```console +dotnet add package CSM.SodaDotNet +``` ## Usage examples @@ -172,37 +173,39 @@ namespace SocrataTest Compilation can be done using [Visual Studio Community Edition](https://www.visualstudio.com/en-us/products/visual-studio-community-vs.aspx). -[NUnit](http://nunit.org/) was used to build and run the test projects. Check out the -[NUnit Test Adapter](https://visualstudiogallery.msdn.microsoft.com/6ab922d0-21c0-4f06-ab5f-4ecd1fe7175d) -to run tests from within Visual Studio. +You can also use the .NET CLI: + +```console +git clone git@github.com:CityofSantaMonica/SODA.NET.git SODA.NET +cd SODA.NET +dotnet build +``` -You can also use the `build.cmd` script, which assumes `msbuild` and `nuget` are available: +## Tests - git clone git@github.com:CityofSantaMonica/SODA.NET.git SODA.NET - cd SODA.NET - .\build.cmd +[NUnit](http://nunit.org/) was used to build and run the test projects. -To create the Nuget package artifacts, pass an extra parameter: +Run tests from the Visual Studio Test Explorer, or using the .NET CLI: - .\build.cmd CreatePackages +```console +dotnet test +``` ## Contributing -Check out the -[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) +Check out the [Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) for more details. ## Copyright and License -Copyright 2017 City of Santa Monica, CA +Copyright 2019 City of Santa Monica, CA -Licensed under the -[MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt) +Licensed under the [MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt) ## Thank you A tremendous amount of inspiration for this project came from the following projects. Thank you! - - [Octokit.net](https://github.com/octokit/octokit.net) - - [soda-java](https://github.com/socrata/soda-java/) - - [soda-dotnet](https://github.com/socrata/soda-dotnet) (defunct?) +- [Octokit.net](https://github.com/octokit/octokit.net) +- [soda-java](https://github.com/socrata/soda-java/) +- [soda-dotnet](https://github.com/socrata/soda-dotnet) (defunct?) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index c7b6dcf..06fb149 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,18 +1,24 @@ +### New in 0.10.0 (Released 2019/10/15) + +New features + +- Support for .NET Core 3.0 + ### New in 0.9.0 (Released 2019/07/24) New features - - Support for .NET Core 2.2 +- Support for .NET Core 2.2 ### New in 0.8.0 (Released 2017/12/06) Bug fixes - - Remove discontinued field from `ResourceMetadata` that was causing serialization issues [#69](https://github.com/CityofSantaMonica/SODA.NET/issues/69) +- Remove discontinued field from `ResourceMetadata` that was causing serialization issues [#69](https://github.com/CityofSantaMonica/SODA.NET/issues/69) New features - - `EwsClient` can be instantiated directly, add new implementation `Office365EwsClient` for connecting to Office 365 based accounts [#71](https://github.com/CityofSantaMonica/SODA.NET/pull/71) +- `EwsClient` can be instantiated directly, add new implementation `Office365EwsClient` for connecting to Office 365 based accounts [#71](https://github.com/CityofSantaMonica/SODA.NET/pull/71) ### New in 0.7.0 (Released 2017/01/12) @@ -20,112 +26,112 @@ New features Security enhancements - - Disable security protocols lower than TLS 1.1 [#65](https://github.com/CityofSantaMonica/SODA.NET/issues/65) +- Disable security protocols lower than TLS 1.1 [#65](https://github.com/CityofSantaMonica/SODA.NET/issues/65) Bug fixes - - `SoqlQuery` no longer inserts default query arguments [#59](https://github.com/CityofSantaMonica/SODA.NET/issues/59) +- `SoqlQuery` no longer inserts default query arguments [#59](https://github.com/CityofSantaMonica/SODA.NET/issues/59) New features - - `SoqlQuery` supports the `$having` clause [#60](https://github.com/CityofSantaMonica/SODA.NET/issues/60) - - `SoqlQuery` supports the `$query` clause [#63](https://github.com/CityofSantaMonica/SODA.NET/issues/63) +- `SoqlQuery` supports the `$having` clause [#60](https://github.com/CityofSantaMonica/SODA.NET/issues/60) +- `SoqlQuery` supports the `$query` clause [#63](https://github.com/CityofSantaMonica/SODA.NET/issues/63) ### New in 0.6.0 (Released 2016/06/02) Bug fixes - - `SodaClient` no longer requires an AppToken [#49](https://github.com/CityofSantaMonica/SODA.NET/issues/49) - - `Query` methods behave as advertised and return full result sets [#56](https://github.com/CityofSantaMonica/SODA.NET/issues/56) +- `SodaClient` no longer requires an AppToken [#49](https://github.com/CityofSantaMonica/SODA.NET/issues/49) +- `Query` methods behave as advertised and return full result sets [#56](https://github.com/CityofSantaMonica/SODA.NET/issues/56) Deprecation Notices - - `ExcelOleDbHelper` in `SODA.Utilities` was deprecated in `v0.5.0` and has now been removed. +- `ExcelOleDbHelper` in `SODA.Utilities` was deprecated in `v0.5.0` and has now been removed. New features - - `SodaClient` can issue queries directly [#54](https://github.com/CityofSantaMonica/SODA.NET/issues/54) +- `SodaClient` can issue queries directly [#54](https://github.com/CityofSantaMonica/SODA.NET/issues/54) ### New in 0.5.0 (Released 2015/08/12) Dependency Updates - - `Newtonsoft.Json` upgraded to 7.0.1 [#43](https://github.com/CityofSantaMonica/SODA.NET/issues/43) +- `Newtonsoft.Json` upgraded to 7.0.1 [#43](https://github.com/CityofSantaMonica/SODA.NET/issues/43) Deprecation Notices - - `ExcelOleDbHelper` in `SODA.Utilities` has been deprecated and replaced by `ExcelDataReaderHelper`. `ExcelOleDbHelper` will be removed in `v0.6.0`. +- `ExcelOleDbHelper` in `SODA.Utilities` has been deprecated and replaced by `ExcelDataReaderHelper`. `ExcelOleDbHelper` will be removed in `v0.6.0`. New features - - `ExcelDataReaderHelper` for reading data from Excel documents [#46](https://github.com/CityofSantaMonica/SODA.NET/pull/46) via @allejo - - `Ews2010Sp2Client` for utilizing EWS against Exchange 2010 SP2 +- `ExcelDataReaderHelper` for reading data from Excel documents [#46](https://github.com/CityofSantaMonica/SODA.NET/pull/46) via @allejo +- `Ews2010Sp2Client` for utilizing EWS against Exchange 2010 SP2 ### New in 0.4.1 (Released 2015/06/11) Bug fixes - - `SoqlQuery.MaximumLimit` increased to 50K [#39](https://github.com/CityofSantaMonica/SODA.NET/issues/39) via @chrismetcalf +- `SoqlQuery.MaximumLimit` increased to 50K [#39](https://github.com/CityofSantaMonica/SODA.NET/issues/39) via @chrismetcalf ### New in 0.4.0 (Released 2015/05/07) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.8 +- `Newtonsoft.Json` upgraded to 6.0.8 New features - - Overload to skip header for `SeparatedValuesSerializer` [#31](https://github.com/CityofSantaMonica/SODA.NET/issues/31) - - Optional `RequestTimeout` property on `SodaClient` [#28](https://github.com/CityofSantaMonica/SODA.NET/issues/28) +- Overload to skip header for `SeparatedValuesSerializer` [#31](https://github.com/CityofSantaMonica/SODA.NET/issues/31) +- Optional `RequestTimeout` property on `SodaClient` [#28](https://github.com/CityofSantaMonica/SODA.NET/issues/28) ### New in 0.3.0 (Released 2015/01/08) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.7 - - `NUnit` (for test projects) upgraded to 2.6.4 - +- `Newtonsoft.Json` upgraded to 6.0.7 +- `NUnit` (for test projects) upgraded to 2.6.4 + New features - - Implementation of SODA [`PhoneColumn`](https://support.socrata.com/hc/en-us/articles/202949918-Importing-Data-Types-and-You-) [#24](https://github.com/CityofSantaMonica/SODA.NET/pull/24) via @mickmorbitzer - - Added a Uri helper for Foundry-style API documentation pages [#22](https://github.com/CityofSantaMonica/SODA.NET/issues/22) +- Implementation of SODA [`PhoneColumn`](https://support.socrata.com/hc/en-us/articles/202949918-Importing-Data-Types-and-You-) [#24](https://github.com/CityofSantaMonica/SODA.NET/pull/24) via @mickmorbitzer +- Added a Uri helper for Foundry-style API documentation pages [#22](https://github.com/CityofSantaMonica/SODA.NET/issues/22) ### New in 0.2.1 (Released 2014/12/02) Bug fixes - - - Soql Limit and Offset throw exceptions for out of range values [#18](https://github.com/CityofSantaMonica/SODA.NET/issues/18) - - Fixed column aliasing bug using Soql As [#17](https://github.com/CityofSantaMonica/SODA.NET/issues/17) + +- Soql Limit and Offset throw exceptions for out of range values [#18](https://github.com/CityofSantaMonica/SODA.NET/issues/18) +- Fixed column aliasing bug using Soql As [#17](https://github.com/CityofSantaMonica/SODA.NET/issues/17) ### New in 0.2.0 (Released 2014/11/03) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.6 [#15](https://github.com/CityofSantaMonica/SODA.NET/issues/15) +- `Newtonsoft.Json` upgraded to 6.0.6 [#15](https://github.com/CityofSantaMonica/SODA.NET/issues/15) New features - - Convenience `Query` method on `Resource` for results that are collections of `TRow` [#12](https://github.com/CityofSantaMonica/SODA.NET/issues/12) - - `SeparatedValuesSerializer` in `SODA.Utilities` can serialize entities to CSV/TSV strings in memory [#2](https://github.com/CityofSantaMonica/SODA.NET/issues/2) +- Convenience `Query` method on `Resource` for results that are collections of `TRow` [#12](https://github.com/CityofSantaMonica/SODA.NET/issues/12) +- `SeparatedValuesSerializer` in `SODA.Utilities` can serialize entities to CSV/TSV strings in memory [#2](https://github.com/CityofSantaMonica/SODA.NET/issues/2) ### New in 0.1.2 (Released 2014/10/01) Minor bug fixes and cleanup - - `SodaResult` members aren't publicly settable [#9](https://github.com/CityofSantaMonica/SODA.NET/issues/9) - - `unwrapExceptionMessage` moved to a public extension of `WebException` [#8](https://github.com/CityofSantaMonica/SODA.NET/issues/8) - - `Newtonsoft.Json` upgraded to 6.0.5 [#6](https://github.com/CityofSantaMonica/SODA.NET/issues/6) - - `SodaResult` correctly deserializes the `By RowIdentifier` value [#5](https://github.com/CityofSantaMonica/SODA.NET/issues/5) - - Some common assembly information [moved to a shared solution-level file](https://github.com/CityofSantaMonica/SODA.NET/commit/5cf686018b49fcd7883561b8a37ec214246d07e6). This will help with deployment +- `SodaResult` members aren't publicly settable [#9](https://github.com/CityofSantaMonica/SODA.NET/issues/9) +- `unwrapExceptionMessage` moved to a public extension of `WebException` [#8](https://github.com/CityofSantaMonica/SODA.NET/issues/8) +- `Newtonsoft.Json` upgraded to 6.0.5 [#6](https://github.com/CityofSantaMonica/SODA.NET/issues/6) +- `SodaResult` correctly deserializes the `By RowIdentifier` value [#5](https://github.com/CityofSantaMonica/SODA.NET/issues/5) +- Some common assembly information [moved to a shared solution-level file](https://github.com/CityofSantaMonica/SODA.NET/commit/5cf686018b49fcd7883561b8a37ec214246d07e6). This will help with deployment ### New in 0.1.1 (Released 2014/09/16) Minor bug fixes and cleanup - - Replace with CSV correctly parses `SodaResult` [#4](https://github.com/CityofSantaMonica/SODA.NET/issues/4) - - Upsert with CSV correctly parses `SodaResult` [#3](https://github.com/CityofSantaMonica/SODA.NET/issues/3) +- Replace with CSV correctly parses `SodaResult` [#4](https://github.com/CityofSantaMonica/SODA.NET/issues/4) +- Upsert with CSV correctly parses `SodaResult` [#3](https://github.com/CityofSantaMonica/SODA.NET/issues/3) ### New in 0.1.0 (Released 2014/08/28) - - Initial release! - - This library is under active development. As such, pre-v1.0 versions may introduce breaking changes until things stabilize around v1.0. Every effort will be made to ensure that any breaking change is well documented and that appropriate workarounds are suggested. +- Initial release! +- This library is under active development. As such, pre-v1.0 versions may introduce breaking changes until things stabilize around v1.0. Every effort will be made to ensure that any breaking change is well documented and that appropriate workarounds are suggested. diff --git a/SODA.sln b/SODA.sln index d075090..6f5e8b6 100644 --- a/SODA.sln +++ b/SODA.sln @@ -34,6 +34,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NetCore22.SODA.Tests", "Net EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NetCore22.Utilities.Tests", "NetCore22.Utilities.Tests\NetCore22.Utilities.Tests.csproj", "{02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCore30.SODA.Tests", "NetCore30.SODA.Tests\NetCore30.SODA.Tests.csproj", "{0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCore30.Utilities.Tests", "NetCore30.Utilities.Tests\NetCore30.Utilities.Tests.csproj", "{38EAAD87-8263-45A4-AE30-A923EC44589E}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution SODA.Tests\SODA.Tests.projitems*{1112171d-6a3f-49f6-818a-fb44195ac837}*SharedItemsImports = 13 @@ -70,6 +74,14 @@ Global {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Debug|Any CPU.Build.0 = Debug|Any CPU {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Release|Any CPU.ActiveCfg = Release|Any CPU {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Release|Any CPU.Build.0 = Release|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Release|Any CPU.Build.0 = Release|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -81,6 +93,8 @@ Global {763FFFCB-49DD-4A86-B5A1-B51EA15CE764} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} {CFAEADE7-EBEA-40F8-9630-C699AA5F8274} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} + {38EAAD87-8263-45A4-AE30-A923EC44589E} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {17A9746D-4584-4AD5-B2FA-2A9EDFC6ED09} diff --git a/SODA/Models/LocationColumn.cs b/SODA/Models/LocationColumn.cs index 6ec3b7e..f14ef81 100644 --- a/SODA/Models/LocationColumn.cs +++ b/SODA/Models/LocationColumn.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using SODA.Utilities; diff --git a/SODA/Models/PhoneColumn.cs b/SODA/Models/PhoneColumn.cs index 02259cf..63e3abf 100644 --- a/SODA/Models/PhoneColumn.cs +++ b/SODA/Models/PhoneColumn.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Newtonsoft.Json; diff --git a/SODA/Properties/AssemblyInfo.cs b/SODA/Properties/AssemblyInfo.cs index 179e20d..c16fcb4 100644 --- a/SODA/Properties/AssemblyInfo.cs +++ b/SODA/Properties/AssemblyInfo.cs @@ -2,3 +2,4 @@ [assembly: InternalsVisibleTo("Net45.SODA.Tests")] [assembly: InternalsVisibleTo("NetCore22.SODA.Tests")] +[assembly: InternalsVisibleTo("NetCore30.SODA.Tests")] diff --git a/SODA/Resource.cs b/SODA/Resource.cs index d02252a..da05995 100644 --- a/SODA/Resource.cs +++ b/SODA/Resource.cs @@ -3,8 +3,7 @@ using System.Linq; using System.Runtime.CompilerServices; using SODA.Utilities; -[assembly: InternalsVisibleTo("Net45.Tests")] -[assembly: InternalsVisibleTo("NetCore22.Tests")] + namespace SODA { /// diff --git a/SODA/ResourceMetadata.cs b/SODA/ResourceMetadata.cs index 12cfd85..0c7acc5 100644 --- a/SODA/ResourceMetadata.cs +++ b/SODA/ResourceMetadata.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using SODA.Utilities; diff --git a/SODA/SodaRequest.cs b/SODA/SodaRequest.cs index c9c96c1..7f5b2fd 100644 --- a/SODA/SodaRequest.cs +++ b/SODA/SodaRequest.cs @@ -3,6 +3,7 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Runtime.CompilerServices; using System.Text; using System.Xml; using System.Xml.Serialization; diff --git a/Utilities/DataFileExporter.cs b/Utilities/DataFileExporter.cs index 091cb69..196b003 100644 --- a/Utilities/DataFileExporter.cs +++ b/Utilities/DataFileExporter.cs @@ -3,9 +3,6 @@ using System.IO; using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Utilities.Net45.Tests")] -[assembly: InternalsVisibleTo("Utilities.NetCore22.Tests")] - namespace SODA.Utilities { /// diff --git a/Utilities/Properties/AssemblyInfo.cs b/Utilities/Properties/AssemblyInfo.cs index f81c691..cdde310 100644 --- a/Utilities/Properties/AssemblyInfo.cs +++ b/Utilities/Properties/AssemblyInfo.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Net45.SODA.Utilities.Tests")] -[assembly: InternalsVisibleTo("NetCore22.SODA.Utilities.Tests")] \ No newline at end of file +[assembly: InternalsVisibleTo("NetCore22.SODA.Utilities.Tests")] +[assembly: InternalsVisibleTo("NetCore30.SODA.Utilities.Tests")] \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index d7247cd..4a3d82b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ -version: 0.9.0-{build} +version: 0.10.1-{build} -image: Visual Studio 2017 +image: Visual Studio 2019 pull_requests: do_not_increment_build_number: true @@ -15,3 +15,6 @@ build_script: test_script: - dotnet test + +notifications: + - provider: GitHubPullRequest \ No newline at end of file diff --git a/icon.png b/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..f46a15e7acc8f47b6d9ddc05d2a47946aa095933 GIT binary patch literal 3452 zcmV-?4TJKDP)#= z6crN`78Dj26&D#67#bKF8yFiL8XO)RA0QndA|NCtCMql>DJv;2F)cDRD=;%LHaRmm zIyO5!IX*!=K|}$0VF7w#0(xQsd0_>6V+np{K}SkNOHN5nQAv{Q7n zTYb7$dAB*0fkB#wd4Ys|g@{$9lUb&gVyvBBfV^RXy=8^JX|JMkw5Vx_!ETGgXNJIW zj>UA4#&?s+dX~z5n#_He%YlcBf{2WTijRkjkcp0wi;+ke8X1o1K@Q zpqQVbpQWapqN9esv7n`=qNk~&sH&u?tEj84tgo`NwzsjfwY9psv$(kGg@Wsji|UAl z?3a`5nU(FFnC_yV@2RHnt*h{`t?;z4@w>M1zPXmfyN|)Qy1>8k!o7r^&xWDUi=@(z zsMM0F)tau^m#o*HwcDY$+oZbPrMumzzTTe5z@y8OW%FE2p$jZ#k&e6}%)63i0($dw`)YsP5+1c6K z+S=UR+~3vT-txu5^UTci$;b54()QZi_1DDu1y(bwqU z-{Rcj;@;u!;^F1w+SjG>E-A1;p6b`?(pmE z_3-cU`|a@W@b&TV^78TX^YZld_V@Mm_w@7i{Pp(y^7QiZ`SL;#2d9Y_EG010qNS#tmYE+YT{E+YYWr9XB601606L_t(|+U?s5R1;Sk z0N`96O4aJRthm14!t$yTv4YsDUAOfybz)GVLakuY)kRQ2K}ArsY^`mzYU?VBNPWthAKmioOejt=S&PQIq#&+>A|yAJdaPJP6e<&l#^ z>u}tw;KlN2lEbSZ+#%z^^3utHHKx{cZc$8Aronl~zV!5~JDr6snCS|xE&rlgpSV{B zn;t(ZtyiSKuh9tia%`!mddTTItk~^5K{(-Emt7UVYWmEHYIu9`_r3eBjV z&8g+uY^W}mJ^LJ0S4H@prqlLuJJ+Q*XJK;1_1W)zu&-3HM>DJ@RB(K4Y?|0ysMo2F zH(D+N4lW9gEi%nSH{r``dtK;*%0L$+9o8>X9JIv5dd@9>)n@rp!MSChkexEARL{0a zQg2JsbXJO!9_#Po6Q3_$G;8L}S)VLg`g0kn$J)F8t5-84d|%hBkkWgf&9JcOuMP`~ zkLvWfCsAB&z5I6E)ua=o?1x1&XAIOpl<_ycENB6~)465gAawdhx1>DV{2`OkAq&b) z>D8~MOr(~XU*FIzC%=FH5sRcF7S%oVZ>74q_ZuuUltD3|MV z3}`{6(!c+kVflH+pF0``vsC@w@gO_%+Q`K-c&;qf(aJa)ql{yxlo1ynpOA1aBlAvH zQAuS(gPb4;E-VeVQDkRlWSmQgk2`iW>d?VeD}0vB7mIKVn>%+dhT$Ud0ypnv|A>q_ zb}qZHy20?Qw)%*uDap!+k2$<*nVT4?5DBMnm}sHbvI9|Z8AUa6f`@vMm)toUeZXgd zNF~CRDr}zHiX#by4FoT>OFYevi}Z2DH6TR#c>ap$>>7fnaU?G~8|gid?!m-=|FKBU z+nBD-j9M3vmErS~B=sGn`^p3oa_* zZ7s!E*iqq4@MIy5wWeS(JYA2f58|yuT*8m#&JwIOrJa{)LtCm3i(aANt|dHJ&a^Pmn|mZ1-JV(cK z>SZBH&a4!_&VeKg&*=PARc8y)kbT4pe?0`p#8K6{|B5o$VD6qXbXmGAU6w8j^CPMCzvR{BHgaYu z4;kl?3tt2wPZisatLKz)Wf6=G&%av?WzQg#j%VA^s+O}p`XW%3olQ{YXC4Rz0VEly z>XbBF+i((w^qnK+%HnZZ`BF9xlI1osNZRD(*(tX)j?S&Kl0ccIl;q0dd9m37rMBqJ z_Rx?B<(2|V^dSU7O95wn6zkbyu|aR1SzA6=Z5cztuy9T-jW(-OPUa~sg3+lYB&}`K zquhRL%E*VDT3}sbV8|0FKwU*9hN994ico)16&`#ay2 zJ-WCqD3*LPEU`gbwr$zOt)*5=L$(F>=hwVb%d}LjwOzM<(}-MMJ5=IlZEd{~!c_EP z7pmnza!t>-Ls-ADdopa8#PcEJ^rV$>DlE4#zVf=R4Pd(rFM{=(v^gg) zc_cB@ehHn*)aq_(9yQn5+j#sX08Ma$ka5;lX>&%cMg6&dv?&-FhOX6@##{MY`b%0Y z8_=O+VguC{^0LPqd%wGA@(dksW0~qX${#Ir|*LDYo$Piwp2Flv32Xm9Od6@V(D zDsg{ts}d;gj?M+Td!lTmnt&b`0O4gp)m#nsIF$LS!0<&01CIMjVVGaO7G!8{vv?3X4)a`DtsU-2>pWd^``jU|mL6^|5^rT8rXwyy)A zPc1|`<13Z`j72*QQ(6Mh_5w4|vm8D|ZCo({Mi1C5<>f(Ul%*eHrFh!;fo4m$hsa{t zVu5LEYy(UgmdgMbNz|5Dpt7KmkW8IpEC3@l7WC7+wjUUQl8@T(H`Zd=znaYR=mEeV zP?k-=;4~vvi&P2|yMta-hte$MpFlrSZAn&J(9y42MxeYjKEH@MW-DsruC}}+nq7iq zZZ9gK=(u5N)OkpT1(l6w#C>Rk&R0fRn#b>kG#CihLA6ENM`alzW3hw)&;zZC!UPc2 z&xkZCFC>fQ0x;iNmfyD@DoND+J^gwBBYT-F9My|tH~^!bD}SjWYZ@V%PP+M{9F10& zUEY|r#SgkYc(xT;dZUGEN-wa*AN0KiW!q6trHlk%Xc;UX48ZVbP+)Hi{3XFBm2JH3 zkGf%8Ba0>fi>l-e8xpCHf^F-=%OrteWw0(eDJkg+`j=FI_6QD$Kx13BKVVaSZqUYj z^?|ub=L?_&J~>bAmW(z$-yR&iA3mIht e(q;MIwfq;G3D8M~aDRdT0000