diff --git a/.gitignore b/.gitignore index a055142..92d8441 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ Desktop.ini *.suo *.user .vs/ +.vscode/ # Build results bin/[Dd]ebug/ diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 8e20101..4afc6c4 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,6 +1,6 @@ # Deploying a new release -Deployment is hard. There are too many manual steps. Let's at least enumerate +Deployment is hard. There are too many manual steps. Let's at least enumerate them so everyone is on the same page. Assuming all changes for the new release are commited, tests have been run and @@ -9,11 +9,11 @@ are passing, and you're happy with the state of things... 1. Create a branch named `release` 1. Update [`ReleaseNotes.md`](ReleaseNotes.md), following the existing format 1. Bump the version number in [`Directory.build.props`](Directory.build.props) and [`appveyor.yml`](appveyor.yml) -1. Push the branch to GitHub and create a pull request +1. Create the NuGet packages (in the local solution root) for the new version: `dotnet pack --output .` +1. Test the NuGet packages! [How to install NuGet package locally](http://stackoverflow.com/questions/10240029/how-to-install-a-nuget-package-nupkg-file-locally) +1. Push the `release` branch to GitHub and create a pull request 1. If the build succeeds, accept the pull request 1. Create and push a tag from the new HEAD `git tag v#.#.#` and `git push --tags` -1. Create the NuGet packages (in the local solution root) for the new version: `dotnet pack --output ..\` -1. Test the NuGet packages! [How to install NuGet package locally](http://stackoverflow.com/questions/10240029/how-to-install-a-nuget-package-nupkg-file-locally) 1. [Create a new release](https://help.github.com/articles/creating-releases) using the tag you just created and pasting in the release notes you just wrote up. Attach a copy of the latest `.nupkg` files generated above. 1. Push the new packages up to NuGet `dotnet nuget push` diff --git a/Directory.build.props b/Directory.build.props index bb9a026..431917d 100644 --- a/Directory.build.props +++ b/Directory.build.props @@ -6,6 +6,10 @@ SODA https://github.com/CityofSantaMonica/SODA.NET MIT - 0.9.0 + icon.png + 0.10.1 + + + \ No newline at end of file diff --git a/Net45.SODA.Tests/Net45.SODA.Tests.csproj b/Net45.SODA.Tests/Net45.SODA.Tests.csproj index c08b6f0..950f67c 100644 --- a/Net45.SODA.Tests/Net45.SODA.Tests.csproj +++ b/Net45.SODA.Tests/Net45.SODA.Tests.csproj @@ -9,9 +9,9 @@ - - - + + + diff --git a/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj b/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj new file mode 100644 index 0000000..c8a18c2 --- /dev/null +++ b/NetCore30.SODA.Tests/NetCore30.SODA.Tests.csproj @@ -0,0 +1,23 @@ + + + + netcoreapp3.0 + + false + + NetCore30.SODA.Tests + + + + + + + + + + + + + + + diff --git a/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj b/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj new file mode 100644 index 0000000..6af4b68 --- /dev/null +++ b/NetCore30.Utilities.Tests/NetCore30.Utilities.Tests.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp3.0 + + false + + NetCore30.SODA.Utilities.Tests + + NetCore30.SODA.Utilities.Tests + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index 0af98ec..110fde1 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # SODA.NET [![Build status](https://ci.appveyor.com/api/projects/status/yub6lyl79573lufv/branch/master?svg=true)](https://ci.appveyor.com/project/thekaveman/soda-net/branch/master) -A [Socrata Open Data API](http://dev.socrata.com) (SODA) client library targeting -.NET 4.5 and above. +A [Socrata Open Data API](https://dev.socrata.com) (SODA) client library targeting .NET 4.5 and above. ## Getting Started SODA.NET is available as a [NuGet package](https://www.nuget.org/packages/CSM.SodaDotNet/). - Install-Package CSM.SodaDotNet +```console +dotnet add package CSM.SodaDotNet +``` ## Usage examples @@ -47,7 +48,7 @@ var results = dataset.Query(soql); ```c# //make sure to provide auth credentials! -var client = +var client = new SodaClient("data.smgov.net", "AppToken", "user@domain.com", "password"); //Upsert some data serialized as CSV @@ -59,46 +60,152 @@ IEnumerable payload = GetPayloadData(); client.Upsert(payload, "1234-wxyz"); ``` -Note: This library supports writing directly to datasets with the Socrata Open Data API. For write operations that use -data transformations in the Socrata Data Management Experience (the user interface for creating datasets), use the Socrata -Data Management API. For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#) +**SodaClient** cam also be used for performing Dataset Management API requests + +For more details on when to use SODA vs the Socrata Data Management API, see the [Data Management API documentation](https://socratapublishing.docs.apiary.io/#) + +Creating datasets: +```c# +using System; +using SODA; +using System.Diagnostics; + +namespace SocrataTest +{ + class Program + { + static void Main(string[] args) + { + // Initialize the client + SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); + + // Read in File (or other source) + string filepath = @"C:\Users\{user}\Desktop\test.csv"; + string csv = System.IO.File.ReadAllText(filepath); + + // Create a Dataset - either public or private (default: private) + Revision dataset = pipelineClient.CreateDataset("MyNewDataset", "public"); + + string datasetId = dataset.GetFourFour(); + Console.WriteLine(datasetId); + + Source source = pipelineClient.CreateSource(csv, dataset, SodaDataFormat.CSV, "File"); + SchemaTransforms input = pipelineClient.CreateInputSchema(source); + AppliedTransform output = input.Run(); + output.AwaitCompletion(pipelineClient, status => { }); + + // Check for Errors + if (output.GetErrorCount() > 0) + { + Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); + pipelineClient.ExportErrorRows(@"C:\Users\{user}\Desktop\errors.csv", output); + // Optional Throw new Error... + } + + // Apply the revision to the dataset + PipelineJob job = pipelineClient.Apply(output, dataset); + + // Await the completion of the revision and output the processing log + job.AwaitCompletion(status => Console.WriteLine(status)); + + } + } +} +``` + +Creating update, replace, or delete revisions: +```cs +using System; +using SODA; +using System.Diagnostics; + +namespace SocrataTest +{ + class Program + { + static void Main(string[] args) + { + // Initialize the client + SodaClient pipelineClient = new SodaClient("https://{domain}", "{username}", "{password}"); + + // Read in File (or other source) + string filepath = @"C:\Users\{user}\Desktop\test.csv"; + string csv = System.IO.File.ReadAllText(filepath); + + // CREATING A REVISION + // Create a Revision (either update, replace, or delete) + Revision revision = pipelineClient.CreateRevision("update", "1234-abcd"); + + // Upload the file as a new source + Source newSource = pipelineClient.CreateSource(csv, revision, SodaDataFormat.CSV, "MyNewFile"); + //Console.WriteLine(source.GetSchemaId()); + // Get the schema of the new (latest) source + SchemaTransforms newInput = pipelineClient.CreateInputSchema(newSource); + + // Run the output transforms + AppliedTransform newOutput = newInput.Run(); + + // Transforms are applied asynchronously, so we need to wait for them to complete + newOutput.AwaitCompletion(pipelineClient, status => { }); + + // Check for Errors + if(output.GetErrorCount() > 0) + { + Console.WriteLine(String.Format("ERRORS! {0} row(s) resulted in an error", output.GetErrorCount())); + pipelineClient.ExportErrorRows(@"C:\Users\{user}\Desktop\errors.csv", output); + // Optional Throw new Error... + } + + // Apply the revision to replace/update the dataset + PipelineJob newJob = pipelineClient.Apply(newOutput, revision); + + // Await the completion of the revision and output the processing log + newJob.AwaitCompletion(status => Console.WriteLine(status) ); + + } + } +} + +``` ## Build Compilation can be done using [Visual Studio Community Edition](https://www.visualstudio.com/en-us/products/visual-studio-community-vs.aspx). -[NUnit](http://nunit.org/) was used to build and run the test projects. Check out the -[NUnit Test Adapter](https://visualstudiogallery.msdn.microsoft.com/6ab922d0-21c0-4f06-ab5f-4ecd1fe7175d) -to run tests from within Visual Studio. +You can also use the .NET CLI: -You can also use the `build.cmd` script, which assumes `msbuild` and `nuget` are available: +```console +git clone git@github.com:CityofSantaMonica/SODA.NET.git SODA.NET +cd SODA.NET +dotnet build +``` - git clone git@github.com:CityofSantaMonica/SODA.NET.git SODA.NET - cd SODA.NET - .\build.cmd +## Tests -To create the Nuget package artifacts, pass an extra parameter: +[NUnit](http://nunit.org/) was used to build and run the test projects. - .\build.cmd CreatePackages +Run tests from the Visual Studio Test Explorer, or using the .NET CLI: + +```console +dotnet test +``` ## Contributing -Check out the -[Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) +Check out the [Contributor Guidelines](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/CONTRIBUTING.md) for more details. ## Copyright and License -Copyright 2017 City of Santa Monica, CA +Copyright 2019 City of Santa Monica, CA -Licensed under the -[MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt) +Licensed under the [MIT License](https://github.com/CityOfSantaMonica/SODA.NET/blob/master/LICENSE.txt) ## Thank you A tremendous amount of inspiration for this project came from the following projects. Thank you! - - [Octokit.net](https://github.com/octokit/octokit.net) - - [soda-java](https://github.com/socrata/soda-java/) - - [soda-dotnet](https://github.com/socrata/soda-dotnet) (defunct?) +- [Octokit.net](https://github.com/octokit/octokit.net) +- [soda-java](https://github.com/socrata/soda-java/) +- [soda-dotnet](https://github.com/socrata/soda-dotnet) (defunct?) diff --git a/ReleaseNotes.md b/ReleaseNotes.md index c7b6dcf..06fb149 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,18 +1,24 @@ +### New in 0.10.0 (Released 2019/10/15) + +New features + +- Support for .NET Core 3.0 + ### New in 0.9.0 (Released 2019/07/24) New features - - Support for .NET Core 2.2 +- Support for .NET Core 2.2 ### New in 0.8.0 (Released 2017/12/06) Bug fixes - - Remove discontinued field from `ResourceMetadata` that was causing serialization issues [#69](https://github.com/CityofSantaMonica/SODA.NET/issues/69) +- Remove discontinued field from `ResourceMetadata` that was causing serialization issues [#69](https://github.com/CityofSantaMonica/SODA.NET/issues/69) New features - - `EwsClient` can be instantiated directly, add new implementation `Office365EwsClient` for connecting to Office 365 based accounts [#71](https://github.com/CityofSantaMonica/SODA.NET/pull/71) +- `EwsClient` can be instantiated directly, add new implementation `Office365EwsClient` for connecting to Office 365 based accounts [#71](https://github.com/CityofSantaMonica/SODA.NET/pull/71) ### New in 0.7.0 (Released 2017/01/12) @@ -20,112 +26,112 @@ New features Security enhancements - - Disable security protocols lower than TLS 1.1 [#65](https://github.com/CityofSantaMonica/SODA.NET/issues/65) +- Disable security protocols lower than TLS 1.1 [#65](https://github.com/CityofSantaMonica/SODA.NET/issues/65) Bug fixes - - `SoqlQuery` no longer inserts default query arguments [#59](https://github.com/CityofSantaMonica/SODA.NET/issues/59) +- `SoqlQuery` no longer inserts default query arguments [#59](https://github.com/CityofSantaMonica/SODA.NET/issues/59) New features - - `SoqlQuery` supports the `$having` clause [#60](https://github.com/CityofSantaMonica/SODA.NET/issues/60) - - `SoqlQuery` supports the `$query` clause [#63](https://github.com/CityofSantaMonica/SODA.NET/issues/63) +- `SoqlQuery` supports the `$having` clause [#60](https://github.com/CityofSantaMonica/SODA.NET/issues/60) +- `SoqlQuery` supports the `$query` clause [#63](https://github.com/CityofSantaMonica/SODA.NET/issues/63) ### New in 0.6.0 (Released 2016/06/02) Bug fixes - - `SodaClient` no longer requires an AppToken [#49](https://github.com/CityofSantaMonica/SODA.NET/issues/49) - - `Query` methods behave as advertised and return full result sets [#56](https://github.com/CityofSantaMonica/SODA.NET/issues/56) +- `SodaClient` no longer requires an AppToken [#49](https://github.com/CityofSantaMonica/SODA.NET/issues/49) +- `Query` methods behave as advertised and return full result sets [#56](https://github.com/CityofSantaMonica/SODA.NET/issues/56) Deprecation Notices - - `ExcelOleDbHelper` in `SODA.Utilities` was deprecated in `v0.5.0` and has now been removed. +- `ExcelOleDbHelper` in `SODA.Utilities` was deprecated in `v0.5.0` and has now been removed. New features - - `SodaClient` can issue queries directly [#54](https://github.com/CityofSantaMonica/SODA.NET/issues/54) +- `SodaClient` can issue queries directly [#54](https://github.com/CityofSantaMonica/SODA.NET/issues/54) ### New in 0.5.0 (Released 2015/08/12) Dependency Updates - - `Newtonsoft.Json` upgraded to 7.0.1 [#43](https://github.com/CityofSantaMonica/SODA.NET/issues/43) +- `Newtonsoft.Json` upgraded to 7.0.1 [#43](https://github.com/CityofSantaMonica/SODA.NET/issues/43) Deprecation Notices - - `ExcelOleDbHelper` in `SODA.Utilities` has been deprecated and replaced by `ExcelDataReaderHelper`. `ExcelOleDbHelper` will be removed in `v0.6.0`. +- `ExcelOleDbHelper` in `SODA.Utilities` has been deprecated and replaced by `ExcelDataReaderHelper`. `ExcelOleDbHelper` will be removed in `v0.6.0`. New features - - `ExcelDataReaderHelper` for reading data from Excel documents [#46](https://github.com/CityofSantaMonica/SODA.NET/pull/46) via @allejo - - `Ews2010Sp2Client` for utilizing EWS against Exchange 2010 SP2 +- `ExcelDataReaderHelper` for reading data from Excel documents [#46](https://github.com/CityofSantaMonica/SODA.NET/pull/46) via @allejo +- `Ews2010Sp2Client` for utilizing EWS against Exchange 2010 SP2 ### New in 0.4.1 (Released 2015/06/11) Bug fixes - - `SoqlQuery.MaximumLimit` increased to 50K [#39](https://github.com/CityofSantaMonica/SODA.NET/issues/39) via @chrismetcalf +- `SoqlQuery.MaximumLimit` increased to 50K [#39](https://github.com/CityofSantaMonica/SODA.NET/issues/39) via @chrismetcalf ### New in 0.4.0 (Released 2015/05/07) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.8 +- `Newtonsoft.Json` upgraded to 6.0.8 New features - - Overload to skip header for `SeparatedValuesSerializer` [#31](https://github.com/CityofSantaMonica/SODA.NET/issues/31) - - Optional `RequestTimeout` property on `SodaClient` [#28](https://github.com/CityofSantaMonica/SODA.NET/issues/28) +- Overload to skip header for `SeparatedValuesSerializer` [#31](https://github.com/CityofSantaMonica/SODA.NET/issues/31) +- Optional `RequestTimeout` property on `SodaClient` [#28](https://github.com/CityofSantaMonica/SODA.NET/issues/28) ### New in 0.3.0 (Released 2015/01/08) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.7 - - `NUnit` (for test projects) upgraded to 2.6.4 - +- `Newtonsoft.Json` upgraded to 6.0.7 +- `NUnit` (for test projects) upgraded to 2.6.4 + New features - - Implementation of SODA [`PhoneColumn`](https://support.socrata.com/hc/en-us/articles/202949918-Importing-Data-Types-and-You-) [#24](https://github.com/CityofSantaMonica/SODA.NET/pull/24) via @mickmorbitzer - - Added a Uri helper for Foundry-style API documentation pages [#22](https://github.com/CityofSantaMonica/SODA.NET/issues/22) +- Implementation of SODA [`PhoneColumn`](https://support.socrata.com/hc/en-us/articles/202949918-Importing-Data-Types-and-You-) [#24](https://github.com/CityofSantaMonica/SODA.NET/pull/24) via @mickmorbitzer +- Added a Uri helper for Foundry-style API documentation pages [#22](https://github.com/CityofSantaMonica/SODA.NET/issues/22) ### New in 0.2.1 (Released 2014/12/02) Bug fixes - - - Soql Limit and Offset throw exceptions for out of range values [#18](https://github.com/CityofSantaMonica/SODA.NET/issues/18) - - Fixed column aliasing bug using Soql As [#17](https://github.com/CityofSantaMonica/SODA.NET/issues/17) + +- Soql Limit and Offset throw exceptions for out of range values [#18](https://github.com/CityofSantaMonica/SODA.NET/issues/18) +- Fixed column aliasing bug using Soql As [#17](https://github.com/CityofSantaMonica/SODA.NET/issues/17) ### New in 0.2.0 (Released 2014/11/03) Dependency Updates - - `Newtonsoft.Json` upgraded to 6.0.6 [#15](https://github.com/CityofSantaMonica/SODA.NET/issues/15) +- `Newtonsoft.Json` upgraded to 6.0.6 [#15](https://github.com/CityofSantaMonica/SODA.NET/issues/15) New features - - Convenience `Query` method on `Resource` for results that are collections of `TRow` [#12](https://github.com/CityofSantaMonica/SODA.NET/issues/12) - - `SeparatedValuesSerializer` in `SODA.Utilities` can serialize entities to CSV/TSV strings in memory [#2](https://github.com/CityofSantaMonica/SODA.NET/issues/2) +- Convenience `Query` method on `Resource` for results that are collections of `TRow` [#12](https://github.com/CityofSantaMonica/SODA.NET/issues/12) +- `SeparatedValuesSerializer` in `SODA.Utilities` can serialize entities to CSV/TSV strings in memory [#2](https://github.com/CityofSantaMonica/SODA.NET/issues/2) ### New in 0.1.2 (Released 2014/10/01) Minor bug fixes and cleanup - - `SodaResult` members aren't publicly settable [#9](https://github.com/CityofSantaMonica/SODA.NET/issues/9) - - `unwrapExceptionMessage` moved to a public extension of `WebException` [#8](https://github.com/CityofSantaMonica/SODA.NET/issues/8) - - `Newtonsoft.Json` upgraded to 6.0.5 [#6](https://github.com/CityofSantaMonica/SODA.NET/issues/6) - - `SodaResult` correctly deserializes the `By RowIdentifier` value [#5](https://github.com/CityofSantaMonica/SODA.NET/issues/5) - - Some common assembly information [moved to a shared solution-level file](https://github.com/CityofSantaMonica/SODA.NET/commit/5cf686018b49fcd7883561b8a37ec214246d07e6). This will help with deployment +- `SodaResult` members aren't publicly settable [#9](https://github.com/CityofSantaMonica/SODA.NET/issues/9) +- `unwrapExceptionMessage` moved to a public extension of `WebException` [#8](https://github.com/CityofSantaMonica/SODA.NET/issues/8) +- `Newtonsoft.Json` upgraded to 6.0.5 [#6](https://github.com/CityofSantaMonica/SODA.NET/issues/6) +- `SodaResult` correctly deserializes the `By RowIdentifier` value [#5](https://github.com/CityofSantaMonica/SODA.NET/issues/5) +- Some common assembly information [moved to a shared solution-level file](https://github.com/CityofSantaMonica/SODA.NET/commit/5cf686018b49fcd7883561b8a37ec214246d07e6). This will help with deployment ### New in 0.1.1 (Released 2014/09/16) Minor bug fixes and cleanup - - Replace with CSV correctly parses `SodaResult` [#4](https://github.com/CityofSantaMonica/SODA.NET/issues/4) - - Upsert with CSV correctly parses `SodaResult` [#3](https://github.com/CityofSantaMonica/SODA.NET/issues/3) +- Replace with CSV correctly parses `SodaResult` [#4](https://github.com/CityofSantaMonica/SODA.NET/issues/4) +- Upsert with CSV correctly parses `SodaResult` [#3](https://github.com/CityofSantaMonica/SODA.NET/issues/3) ### New in 0.1.0 (Released 2014/08/28) - - Initial release! - - This library is under active development. As such, pre-v1.0 versions may introduce breaking changes until things stabilize around v1.0. Every effort will be made to ensure that any breaking change is well documented and that appropriate workarounds are suggested. +- Initial release! +- This library is under active development. As such, pre-v1.0 versions may introduce breaking changes until things stabilize around v1.0. Every effort will be made to ensure that any breaking change is well documented and that appropriate workarounds are suggested. diff --git a/SODA.Tests/SodaClientTests.cs b/SODA.Tests/SodaClientTests.cs index 1f569f9..46d7ae5 100644 --- a/SODA.Tests/SodaClientTests.cs +++ b/SODA.Tests/SodaClientTests.cs @@ -266,5 +266,37 @@ public void DeleteRow_Using_Anonymous_Client_Throws_InvalidOperationException() { Assert.That(() => mockClient.DeleteRow(StringMocks.NonEmptyInput, StringMocks.ResourceId), Throws.TypeOf()); } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaClient")] + public void DSMAPI_With_Invalid_ResourceId_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => mockClient.CreateRevision(StringMocks.NonEmptyInput, input), Throws.TypeOf()); + } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaClient")] + public void DSMAPI_With_NULL_Or_Empty_Values_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => mockClient.CreateDataset(input), Throws.TypeOf()); + Assert.That(() => mockClient.CreateRevision(input, StringMocks.ResourceId), Throws.TypeOf()); + Assert.That(() => mockClient.CreateSource(input, null, SodaDataFormat.CSV), Throws.TypeOf()); + Assert.That(() => mockClient.ExportErrorRows(input, null), Throws.TypeOf()); + + } + + [Category("SodaClient")] + public void DSMAPI_With_NUL_Values_Throws_ArgumentOutOfRangeException() + { + Assert.That(() => mockClient.CreateSource(StringMocks.NonEmptyInput, null, SodaDataFormat.CSV), Throws.TypeOf()); + + Assert.That(() => mockClient.GetSource(null), Throws.TypeOf()); + Assert.That(() => mockClient.CreateInputSchema(null), Throws.TypeOf()); + + Assert.That(() => mockClient.ExportErrorRows(StringMocks.NonEmptyInput, null), Throws.TypeOf()); + Assert.That(() => mockClient.Apply(null, null), Throws.TypeOf()); + } } } \ No newline at end of file diff --git a/SODA.Tests/SodaUriTests.cs b/SODA.Tests/SodaUriTests.cs index 3b32cc9..b4bb8ba 100644 --- a/SODA.Tests/SodaUriTests.cs +++ b/SODA.Tests/SodaUriTests.cs @@ -38,6 +38,30 @@ public void All_Methods_Return_Uri_With_Socrata_Domain_As_Host() uri = null; uri = SodaUri.ForCategoryPage(StringMocks.Host, StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForRevision(StringMocks.Host, StringMocks.ResourceId); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForSource(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForUpload(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForApply(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForJob(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); + + uri = null; + uri = SodaUri.ForErrorRows(StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(StringMocks.Host, uri.Host); } [Test] @@ -98,6 +122,30 @@ public void All_Methods_Return_Uri_Using_HTTPS() uri = null; uri = SodaUri.ForCategoryPage("http://" + StringMocks.Host, StringMocks.NonEmptyInput); StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForRevision("http://" + StringMocks.Host, StringMocks.ResourceId); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForSource("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForUpload("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForApply("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForJob("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); + + uri = null; + uri = SodaUri.ForErrorRows("http://" + StringMocks.Host, "/" + StringMocks.NonEmptyInput); + StringAssert.AreEqualIgnoringCase(Uri.UriSchemeHttps, uri.Scheme); } [TestCase(StringMocks.NullInput)] @@ -328,6 +376,30 @@ public void ForCategoryPage_With_Empty_Category_Throws_ArgumentException(string Assert.That(() => SodaUri.ForCategoryPage(StringMocks.Host, input), Throws.TypeOf()); } + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaUri")] + public void DSMAPI_With_Invalid_Endpoint_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => SodaUri.ForRevision(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForUpload(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForSource(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForJob(StringMocks.Host, input), Throws.TypeOf()); + Assert.That(() => SodaUri.ForApply(StringMocks.Host, input), Throws.TypeOf()); + } + + [TestCase(StringMocks.NullInput)] + [TestCase(StringMocks.EmptyInput)] + [Category("SodaUri")] + public void DSMAPI_With_Invalid_Host_Throws_ArgumentOutOfRangeException(string input) + { + Assert.That(() => SodaUri.ForRevision(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForUpload(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForSource(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForJob(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + Assert.That(() => SodaUri.ForApply(input, StringMocks.NonEmptyInput), Throws.TypeOf()); + } + [Test] [Category("SodaUri")] public void ForCategoryPage_With_Valid_Arguments_Creates_CategoryPage_Uri() @@ -352,4 +424,4 @@ public void ForCategoryPage_With_Complex_Category_Uri_Doesnt_Escape_Complex_Cate StringAssert.AreNotEqualIgnoringCase(String.Format("/categories/{0}", Uri.EscapeDataString(complexCategory)), uri.LocalPath); } } -} \ No newline at end of file +} diff --git a/SODA.sln b/SODA.sln index d075090..6f5e8b6 100644 --- a/SODA.sln +++ b/SODA.sln @@ -34,6 +34,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NetCore22.SODA.Tests", "Net EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NetCore22.Utilities.Tests", "NetCore22.Utilities.Tests\NetCore22.Utilities.Tests.csproj", "{02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCore30.SODA.Tests", "NetCore30.SODA.Tests\NetCore30.SODA.Tests.csproj", "{0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetCore30.Utilities.Tests", "NetCore30.Utilities.Tests\NetCore30.Utilities.Tests.csproj", "{38EAAD87-8263-45A4-AE30-A923EC44589E}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution SODA.Tests\SODA.Tests.projitems*{1112171d-6a3f-49f6-818a-fb44195ac837}*SharedItemsImports = 13 @@ -70,6 +74,14 @@ Global {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Debug|Any CPU.Build.0 = Debug|Any CPU {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Release|Any CPU.ActiveCfg = Release|Any CPU {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF}.Release|Any CPU.Build.0 = Release|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F}.Release|Any CPU.Build.0 = Release|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38EAAD87-8263-45A4-AE30-A923EC44589E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -81,6 +93,8 @@ Global {763FFFCB-49DD-4A86-B5A1-B51EA15CE764} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} {CFAEADE7-EBEA-40F8-9630-C699AA5F8274} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} {02CEA30A-5D25-4F9D-B74E-7CED1EBA84BF} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} + {0A4AA36D-8AF5-41B1-8448-06ED13C97B3F} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} + {38EAAD87-8263-45A4-AE30-A923EC44589E} = {76D1DD9D-B476-4900-A561-18BB7CBAF54F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {17A9746D-4584-4AD5-B2FA-2A9EDFC6ED09} diff --git a/SODA/AppliedTransform.cs b/SODA/AppliedTransform.cs new file mode 100644 index 0000000..696ac01 --- /dev/null +++ b/SODA/AppliedTransform.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; +using System; +using SODA.Utilities; + +namespace SODA +{ + /// + /// A class for Applying Transforms. + /// + public class AppliedTransform + { + /// + /// Source object. + /// + Source source; + + /// + /// Create an applied transform object based off a source. + /// + /// AppliedTransform + public AppliedTransform(Source source) + { + this.source = source; + } + + /// + /// Retrieve the Output schema id. + /// + /// Error count + public string GetOutputSchemaId() + { + return this.source.GetSchemaId(); + } + + /// + /// Retrieve Input schema ID. + /// + /// Error count + public string GetInputSchemaId() + { + return this.source.GetInputSchemaId(); + } + + /// + /// Retrieve the error count. + /// + /// Error count + public int GetErrorCount() + { + return this.source.GetErrorCount(); + } + + /// + /// Retrieve the error endpoint. + /// + /// Error endpoint + public string GetErrorRowEndpoint() + { + return this.source.GetErrorRowEndPoint(); + } + + /// + /// Await completion of transforms. + /// + /// The current Soda client + /// Lambda output + public void AwaitCompletion(SodaClient client, Action lambda) + { + this.source = client.GetSource(this.source); + while (!this.source.IsComplete(lambda)) + { + this.source = client.GetSource(this.source); + System.Threading.Thread.Sleep(1000); + } + } + } +} diff --git a/SODA/Models/LocationColumn.cs b/SODA/Models/LocationColumn.cs index 6ec3b7e..f14ef81 100644 --- a/SODA/Models/LocationColumn.cs +++ b/SODA/Models/LocationColumn.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using SODA.Utilities; diff --git a/SODA/Models/PhoneColumn.cs b/SODA/Models/PhoneColumn.cs index 02259cf..63e3abf 100644 --- a/SODA/Models/PhoneColumn.cs +++ b/SODA/Models/PhoneColumn.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Newtonsoft.Json; diff --git a/SODA/PipelineJob.cs b/SODA/PipelineJob.cs new file mode 100644 index 0000000..0894c22 --- /dev/null +++ b/SODA/PipelineJob.cs @@ -0,0 +1,64 @@ +using System; +using SODA.Utilities; +using System.Net; + +namespace SODA +{ + /// + /// A class representing the Data Pipeline job for revisions that have been submitted to Socrata. + /// + public class PipelineJob + { + /// + /// Socrata username. + /// + public string username; + /// + /// Socrata password. + /// + private string password; + /// + /// The revision endpoint. + /// + public Uri revisionEndpoint { get; set; } + + /// + /// Create the pipeline job object. + /// + /// the JobURI. + /// username. + /// Password. + public PipelineJob(Uri jobUri, string user, string pass) + { + username = user; + password = pass; + revisionEndpoint = jobUri; + } + + /// + /// Await the completion of the update, optionally output the status. + /// + /// A lambda function for outputting status if desired. + public void AwaitCompletion(Action lambda) + { + string status = ""; + Result r = null; + while(status != "successful" && status != "failure") + { + var jobRequest = new SodaRequest(revisionEndpoint, "GET", null, username, password, SodaDataFormat.JSON); + try + { + r = jobRequest.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + r = new Result() { Message = webException.Message, IsError = true, ErrorCode = message }; + } + status = r.Resource["task_sets"][0]["status"]; + lambda(status); + System.Threading.Thread.Sleep(1000); + } + } + } +} diff --git a/SODA/Properties/AssemblyInfo.cs b/SODA/Properties/AssemblyInfo.cs index 179e20d..c16fcb4 100644 --- a/SODA/Properties/AssemblyInfo.cs +++ b/SODA/Properties/AssemblyInfo.cs @@ -2,3 +2,4 @@ [assembly: InternalsVisibleTo("Net45.SODA.Tests")] [assembly: InternalsVisibleTo("NetCore22.SODA.Tests")] +[assembly: InternalsVisibleTo("NetCore30.SODA.Tests")] diff --git a/SODA/Resource.cs b/SODA/Resource.cs index d02252a..da05995 100644 --- a/SODA/Resource.cs +++ b/SODA/Resource.cs @@ -3,8 +3,7 @@ using System.Linq; using System.Runtime.CompilerServices; using SODA.Utilities; -[assembly: InternalsVisibleTo("Net45.Tests")] -[assembly: InternalsVisibleTo("NetCore22.Tests")] + namespace SODA { /// diff --git a/SODA/ResourceMetadata.cs b/SODA/ResourceMetadata.cs index 12cfd85..0c7acc5 100644 --- a/SODA/ResourceMetadata.cs +++ b/SODA/ResourceMetadata.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.Serialization; using SODA.Utilities; diff --git a/SODA/Result.cs b/SODA/Result.cs new file mode 100644 index 0000000..7b9ecfd --- /dev/null +++ b/SODA/Result.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; +using SODA.Utilities; + +namespace SODA +{ + /// + /// Gets the information related to the Result of API calls. + /// + [DataContract] + public class Result + { + /// + /// Gets the number of errors. + /// + [DataMember(Name = "Errors")] + public int Errors { get; private set; } + + + /// + /// Gets the explanatory text about this result. + /// + [DataMember(Name = "message")] + public string Message { get; internal set; } + + /// + /// Gets a flag indicating if one or more errors occured. + /// + [DataMember(Name = "error")] + public bool IsError { get; internal set; } + + /// + /// Gets data about any errors that occured. + /// + [DataMember(Name = "code")] + public string ErrorCode { get; internal set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "data")] + public dynamic Data { get; internal set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "resource")] + public Dictionary Resource { get; set; } + + /// + /// Gets any additional data associated with this result. + /// + [DataMember(Name = "links")] + public Dictionary Links { get; set; } + + } +} diff --git a/SODA/Revision.cs b/SODA/Revision.cs new file mode 100644 index 0000000..0569945 --- /dev/null +++ b/SODA/Revision.cs @@ -0,0 +1,69 @@ +using System; +using SODA.Utilities; +using System.Security.Permissions; + +namespace SODA + +{ + /// + /// A class for accessing the revision object. + /// + public class Revision + { + /// + /// The result of a revision being created. + /// + Result result; + + /// + /// A class for handling revisions. + /// + public Revision(Result result) + { + this.result = result; + Console.WriteLine(String.Format("Revision number {0} created", result.Resource["revision_seq"])); + + } + + /// + /// Get the current revision number. + /// + public long GetRevisionNumber() + { + return this.result.Resource["revision_seq"]; + } + + /// + /// Get the dataset ID of the current revision. + /// + public string GetFourFour() + { + return this.result.Resource["fourfour"]; + } + + /// + /// Get the revision endpoint. + /// + public string getRevisionLink() + { + return this.result.Links["show"]; + } + + /// + /// Get the create source link endpoint. + /// + public string GetSourceEndpoint() + { + return this.result.Links["create_source"]; + } + + /// + /// Get the apply link endpoint + /// + public string GetApplyEndpoint() + { + return this.result.Links["apply"]; + } + + } +} diff --git a/SODA/SODA.csproj b/SODA/SODA.csproj index 78d1060..28ae8ed 100644 --- a/SODA/SODA.csproj +++ b/SODA/SODA.csproj @@ -8,6 +8,7 @@ true CSM.SodaDotNet API OpenData Socrata SODA + true diff --git a/SODA/SchemaTransforms.cs b/SODA/SchemaTransforms.cs new file mode 100644 index 0000000..fac591f --- /dev/null +++ b/SODA/SchemaTransforms.cs @@ -0,0 +1,81 @@ +using System.Collections.Generic; +using System; +namespace SODA +{ + /// + /// A class for applying transforms to the schema programmatically. + /// + public class SchemaTransforms + { + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + Source source; + + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public SchemaTransforms(Source source) + { + this.source = source; + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnFieldname() + { + // TODO: WIP + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnDescription() + { + // TODO: WIP + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnDisplayName() + { + // TODO: WIP + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void ChangeColumnTransform() + { + // TODO: WIP + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public void AddColumn() + { + // TODO: WIP + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public Source GetSource() + { + return this.source; + } + /// + /// A class for interacting with Socrata Data Portals using the Socrata Open Data API. + /// + public string GetOutputSchemaId() + { + return this.source.GetSchemaId(); + } + + /// + /// Apply the transforms; Currently a NO-OP. + /// + public AppliedTransform Run() + { + return new AppliedTransform(this.source); + } + + } +} diff --git a/SODA/SodaClient.cs b/SODA/SodaClient.cs index 9b26f62..c2df476 100644 --- a/SODA/SodaClient.cs +++ b/SODA/SodaClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net; using SODA.Utilities; @@ -32,7 +33,7 @@ public class SodaClient /// Authentication is only necessary when accessing datasets that have been marked as private or when making write requests (PUT, POST, and DELETE). /// See http://dev.socrata.com/docs/authentication.html for more information. /// - public readonly string Username; + public readonly string username; //not publicly readable, can only be set in a constructor private readonly string password; @@ -58,7 +59,7 @@ public SodaClient(string host, string appToken, string username, string password Host = SodaUri.enforceHttps(host); AppToken = appToken; - Username = username; + username = username; this.password = password; } @@ -69,7 +70,7 @@ public SodaClient(string host, string appToken, string username, string password /// The user account that this client will use for Authentication during each request. /// The password for the specified that this client will use for Authentication during each request. /// Thrown if no is provided. - public SodaClient(string host, string username, string password) + public SodaClient(string host, string username, string password) : this(host, null, username, password) { } @@ -116,7 +117,7 @@ public IEnumerable GetMetadataPage(int page) { if (page <= 0) throw new ArgumentOutOfRangeException("page", "Resouce metadata catalogs begin on page 1."); - + var catalogUri = SodaUri.ForMetadataList(Host, page); //an entry of raw data contains some, but not all, of the fields required to populate a ResourceMetadata @@ -142,7 +143,7 @@ public Resource GetResource(string resourceId) where TRow : class { if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - + return new Resource(resourceId, this); } @@ -209,12 +210,12 @@ public SodaResult Upsert(string payload, SodaDataFormat dataFormat, string resou if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId); - var request = new SodaRequest(uri, "POST", AppToken, Username, password, dataFormat, payload); + var request = new SodaRequest(uri, "POST", AppToken, username, password, dataFormat, payload); SodaResult result = null; try @@ -255,7 +256,7 @@ public SodaResult Upsert(IEnumerable payload, string resourceId) whe if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); string json = Newtonsoft.Json.JsonConvert.SerializeObject(payload); @@ -278,7 +279,7 @@ public IEnumerable BatchUpsert(IEnumerable payload, int if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); Queue queue = new Queue(payload); @@ -336,7 +337,7 @@ public IEnumerable BatchUpsert(IEnumerable payload, int if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); //we create a no-op function that returns false for all inputs @@ -365,12 +366,12 @@ public SodaResult Replace(string payload, SodaDataFormat dataFormat, string reso if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId); - var request = new SodaRequest(uri, "PUT", AppToken, Username, password, dataFormat, payload); + var request = new SodaRequest(uri, "PUT", AppToken, username, password, dataFormat, payload); SodaResult result = null; try @@ -411,7 +412,7 @@ public SodaResult Replace(IEnumerable payload, string resourceId) wh if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); string json = Newtonsoft.Json.JsonConvert.SerializeObject(payload); @@ -436,12 +437,12 @@ public SodaResult DeleteRow(string rowId, string resourceId) if (FourByFour.IsNotValid(resourceId)) throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); - if (String.IsNullOrEmpty(Username) || String.IsNullOrEmpty(password)) + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) throw new InvalidOperationException("Write operations require an authenticated client."); var uri = SodaUri.ForResourceAPI(Host, resourceId, rowId); - var request = new SodaRequest(uri, "DELETE", AppToken, Username, password); + var request = new SodaRequest(uri, "DELETE", AppToken, username, password); return request.ParseResponse(); } @@ -456,7 +457,7 @@ public SodaResult DeleteRow(string rowId, string resourceId) internal TResult read(Uri uri, SodaDataFormat dataFormat = SodaDataFormat.JSON) where TResult : class { - var request = new SodaRequest(uri, "GET", AppToken, Username, password, dataFormat, null, RequestTimeout); + var request = new SodaRequest(uri, "GET", AppToken, username, password, dataFormat, null, RequestTimeout); return request.ParseResponse(); } @@ -474,9 +475,237 @@ internal TResult write(Uri uri, string method, TPayload paylo where TPayload : class where TResult : class { - var request = new SodaRequest(uri, method, AppToken, Username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout); + var request = new SodaRequest(uri, method, AppToken, username, password, SodaDataFormat.JSON, payload.ToJsonString(), RequestTimeout); return request.ParseResponse(); } + + /// + /// Create a new dataset with a given name and permission level. + /// + /// The dataset name + /// The permission level of the dataset, can be one of either "public" or "private". + /// A newly created Revision. + public Revision CreateDataset(string name, string permission = "private") + { + if (String.IsNullOrEmpty(name)) + throw new ArgumentException("Dataset name required.", "name"); + + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + var revisionUri = SodaUri.ForRevision(Host); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject metadata = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); + metadata["name"] = name; + action["type"] = "replace"; + action["permission"] = permission; + payload["action"] = action; + payload["metadata"] = metadata; + + var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); + + Result result = null; + try + { + result = request.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new Revision(result); + } + + /// + /// Replace any existing rows with the payload data, using the specified resource identifier. + /// + /// One of update, replace, or delete + /// The identifier (4x4) for a resource on the Socrata host to target. + /// The permission level of the dataset, can be one of either "public" or "private". + /// A newly created Revision. + /// Thrown if the specified does not match the Socrata 4x4 pattern. + public Revision CreateRevision(string method, string resourceId, string permission = "private") + { + + if (String.IsNullOrEmpty(method)) + throw new ArgumentException("Method must be specified either update, replace, or delete.", "method"); + + if (FourByFour.IsNotValid(resourceId)) + throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + var revisionUri = SodaUri.ForRevision(Host, resourceId); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject action = new Newtonsoft.Json.Linq.JObject(); + action["type"] = method; + action["permission"] = permission; + payload["action"] = action; + + var request = new SodaRequest(revisionUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); + + Result result = null; + try + { + result = request.ParseResponse(); + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + return new Revision(result); + } + + /// + /// Creates the source for the specified revision. + /// + /// A string of serialized data. + /// The revision created as part of a create revision step. + /// The format of the data. + /// The filename that should be associated with this upload. + /// A indicating success or failure. + /// Thrown if this SodaDSMAPIClient was initialized without authentication credentials. + public Source CreateSource(string data, Revision revision, SodaDataFormat dataFormat = SodaDataFormat.CSV, string filename = "NewUpload") + { + if (String.IsNullOrEmpty(data)) + throw new ArgumentException("Data must be provided.", "data"); + + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + if (revision == null) + throw new ArgumentException("Revision required.", "revision"); + + var sourceUri = SodaUri.ForSource(Host, revision.GetSourceEndpoint()); + Debug.WriteLine(sourceUri.ToString()); + revision.GetRevisionNumber(); + + // Construct Revision Request body + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject source_type = new Newtonsoft.Json.Linq.JObject(); + Newtonsoft.Json.Linq.JObject parse_option = new Newtonsoft.Json.Linq.JObject(); + source_type["type"] = "upload"; + source_type["filename"] = filename; + parse_option["parse_source"] = true; + payload["source_type"] = source_type; + payload["parse_options"] = parse_option; + + var createSourceRequest = new SodaRequest(sourceUri, "POST", null, username, password, SodaDataFormat.JSON, payload.ToString()); + Result sourceOutput = createSourceRequest.ParseResponse(); + string uploadDataPath = sourceOutput.Links["bytes"]; + var uploadUri = SodaUri.ForUpload(Host, uploadDataPath); + Debug.WriteLine(uploadUri.ToString()); + var fileUploadRequest = new SodaRequest(uploadUri, "POST", null, username, password, dataFormat, data); + fileUploadRequest.SetDataType(SodaDataFormat.JSON); + Result result = fileUploadRequest.ParseResponse(); + return new Source(result); + } + + /// + /// Get the specified source data. + /// + /// The result of the Source creation + /// A The updated Source object + public Source GetSource(Source source) + { + if (source == null) + throw new ArgumentException("Source required.", "source"); + var sourceUri = SodaUri.ForSource(Host, source.Self()); + var sourceUpdateResponse = new SodaRequest(sourceUri, "GET", null, username, password, SodaDataFormat.JSON, ""); + Result result = sourceUpdateResponse.ParseResponse(); + return new Source(result); + } + + /// + /// Create the InputSchema from the source. + /// + /// The result of the Source creation + /// A SchemaTransforms object + public SchemaTransforms CreateInputSchema(Source source) + { + if (source == null) + throw new ArgumentException("Source required.", "source"); + return new SchemaTransforms(source); + } + + /// + /// Export the error rows (if present). + /// + /// The output file (csv) + /// The specified transformed output + public void ExportErrorRows(string filepath, AppliedTransform output) + { + if (String.IsNullOrEmpty(filepath)) + throw new ArgumentException("Filepath must be specified.", "filepath"); + + if (output == null) + throw new ArgumentException("Applied Transform required.", "output"); + + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + var endpoint = output.GetErrorRowEndpoint().Replace("{input_schema_id}", output.GetInputSchemaId()).Replace("{output_schema_id}", output.GetOutputSchemaId()); + var errorRowsUri = SodaUri.ForErrorRows(Host, endpoint); + Debug.WriteLine(errorRowsUri.ToString()); + var downloadRowsRequest = new SodaRequest(errorRowsUri, "GET", null, username, password, SodaDataFormat.CSV, ""); + var result = downloadRowsRequest.ParseResponse(); + System.IO.File.WriteAllText(filepath, result); + } + + /// + /// Apply the source, transforms, and update to the specified dataset. + /// + /// A string of serialized data. + /// A string of serialized data. + /// A for determining success or failure. + public PipelineJob Apply(AppliedTransform outputSchema, Revision revision) + { + if (String.IsNullOrEmpty(username) || String.IsNullOrEmpty(password)) + throw new InvalidOperationException("Write operations require an authenticated client."); + + if (outputSchema == null || revision == null) + throw new InvalidOperationException("Both the output schema and revision are required."); + + Newtonsoft.Json.Linq.JObject payload = new Newtonsoft.Json.Linq.JObject(); + payload["output_schema_id"] = outputSchema.GetOutputSchemaId(); + + var uri = SodaUri.ForSource(Host, revision.GetApplyEndpoint()); + Debug.WriteLine(uri.ToString()); + var applyRequest = new SodaRequest(uri, "PUT", null, username, password, SodaDataFormat.JSON, payload.ToString()); + Result result = null; + try + { + result = applyRequest.ParseResponse(); + + } + catch (WebException webException) + { + string message = webException.UnwrapExceptionMessage(); + result = new Result() { Message = webException.Message, IsError = true, ErrorCode = message, Data = payload }; + } + catch (Exception ex) + { + result = new Result() { Message = ex.Message, IsError = true, ErrorCode = ex.Message, Data = payload }; + } + + return new PipelineJob(SodaUri.ForJob(Host, revision.getRevisionLink()), username, password); + } } } diff --git a/SODA/SodaRequest.cs b/SODA/SodaRequest.cs index db482cb..7f5b2fd 100644 --- a/SODA/SodaRequest.cs +++ b/SODA/SodaRequest.cs @@ -3,6 +3,7 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Runtime.CompilerServices; using System.Text; using System.Xml; using System.Xml.Serialization; @@ -75,6 +76,7 @@ internal SodaRequest(Uri uri, string method, string appToken, string username, s break; case SodaDataFormat.CSV: this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("text/csv")); + this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); break; case SodaDataFormat.XML: this.Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/rdf+xml")); @@ -97,7 +99,11 @@ internal SodaRequest(Uri uri, string method, string appToken, string username, s } } } - + + internal void SetDataType(SodaDataFormat format) + { + dataFormat = format; + } /// /// Send this SodaRequest's webRequest and interpret the response. /// diff --git a/SODA/Source.cs b/SODA/Source.cs new file mode 100644 index 0000000..a7075e4 --- /dev/null +++ b/SODA/Source.cs @@ -0,0 +1,88 @@ +using System; +using SODA.Utilities; +using System.Security.Permissions; + +namespace SODA + +{ + /// + /// A class for accessing the Source object. + /// + public class Source + { + /// + /// The result of a source being created. + /// + Result result; + + /// + /// A class for handling Sources. + /// + public Source(Result result) + { + this.result = result; + } + + /// + /// Get lastest Schema ID. + /// + public string GetSchemaId() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["id"]; + } + + /// + /// Get Input schema ID. + /// + /// Input Schema ID + public string GetInputSchemaId() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["input_schema_id"]; + } + + /// + /// Retrieve the error count. + /// + /// Error count + public int GetErrorCount() + { + return this.result.Resource["schemas"][0]["output_schemas"][0]["error_count"]; + } + + /// + /// Retrieve the error count. + /// + /// Error count + public Boolean IsComplete(Action lambda) + { + string completed_at = this.result.Resource["schemas"][0]["output_schemas"][0]["completed_at"]; + if(String.IsNullOrEmpty(completed_at)) + { + lambda("Working..."); + return false; + } else + { + lambda(String.Format("Completed at: {0}", completed_at)); + return true; + } + } + + /// + /// Error row endpoint. + /// + /// Error row endpoint + public string GetErrorRowEndPoint() + { + return this.result.Links["input_schema_links"]["output_schema_links"]["schema_errors"]; + } + + /// + /// Get the self link. + /// + /// Current endpoint + public string Self() + { + return this.result.Links["show"]; + } + } +} \ No newline at end of file diff --git a/SODA/Utilities/SodaUri.cs b/SODA/Utilities/SodaUri.cs index 82dba3b..037f053 100644 --- a/SODA/Utilities/SodaUri.cs +++ b/SODA/Utilities/SodaUri.cs @@ -203,7 +203,7 @@ public static Uri ForQuery(string socrataHost, string resourceId, SoqlQuery soql return new Uri(queryUrl); } - + /// /// Create a Uri to the landing page of a specified category on the specified Socrata host. /// @@ -222,5 +222,124 @@ public static Uri ForCategoryPage(string socrataHost, string category) return new Uri(url); } + + /// + /// Create a revision Uri the specified resource on the specified Socrata host. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A revision Uri for the specified resource on the specified Socrata host. + public static Uri ForRevision(string socrataHost, string resourceId) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (FourByFour.IsNotValid(resourceId)) + throw new ArgumentOutOfRangeException("resourceId", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}/api/publishing/v1/revision/{1}", enforceHttps(socrataHost), resourceId); + return new Uri(url); + } + + /// + /// Create a revision Uri the specified resource on the specified Socrata host. + /// + /// The Socrata host to target. + /// A revision Uri for the specified resource on the specified Socrata host. + public static Uri ForRevision(string socrataHost) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + string url = String.Format("{0}/api/publishing/v1/revision", enforceHttps(socrataHost)); + return new Uri(url); + } + + /// + /// Create a URI for uploading the source data. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForUpload(string socrataHost, string uploadEndpoint) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(uploadEndpoint)) + throw new ArgumentOutOfRangeException("uploadEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), uploadEndpoint.Replace("\"", "")); + return new Uri(url); + + } + /// + /// URI for the particular source. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForSource(string socrataHost, string sourceEndpoint) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(sourceEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), sourceEndpoint.Replace("\"", "")); + return new Uri(url); + + } + + /// + /// URI for Applying the transforms. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForApply(string socrataHost, string applyEndpoint) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(applyEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), applyEndpoint.Replace("\"", "")); + return new Uri(url); + } + + /// + /// Create a URI for the Job. + /// + /// The Socrata host to target. + /// The identifier (4x4) for a resource on the Socrata host to target. + /// A query Uri for the specified resource on the specified Socrata host. + public static Uri ForJob(string socrataHost, string revisionEndpoint) + { + if (String.IsNullOrEmpty(socrataHost)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + + if (String.IsNullOrEmpty(revisionEndpoint)) + throw new ArgumentOutOfRangeException("sourceEndpoint", "The provided resourceId is not a valid Socrata (4x4) resource identifier."); + + string url = String.Format("{0}{1}", enforceHttps(socrataHost), revisionEndpoint.Replace("\"", "")); + return new Uri(url); + } + + /// + /// Create a URI for exporting the error rows. + /// + /// The Socrata host to target. + /// The error endpoint + /// A error Uri for the specified resource on the specified Socrata host. + public static Uri ForErrorRows(string socrataHost, string errorEndpoint) + { + if (String.IsNullOrEmpty(socrataHost) || String.IsNullOrEmpty(errorEndpoint)) + throw new ArgumentException("socrataHost", "Must provide a Socrata host to target."); + string url = String.Format("{0}{1}", enforceHttps(socrataHost), errorEndpoint); + return new Uri(url); + } } -} \ No newline at end of file +} diff --git a/Utilities/DataFileExporter.cs b/Utilities/DataFileExporter.cs index 091cb69..196b003 100644 --- a/Utilities/DataFileExporter.cs +++ b/Utilities/DataFileExporter.cs @@ -3,9 +3,6 @@ using System.IO; using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Utilities.Net45.Tests")] -[assembly: InternalsVisibleTo("Utilities.NetCore22.Tests")] - namespace SODA.Utilities { /// diff --git a/Utilities/Properties/AssemblyInfo.cs b/Utilities/Properties/AssemblyInfo.cs index f81c691..cdde310 100644 --- a/Utilities/Properties/AssemblyInfo.cs +++ b/Utilities/Properties/AssemblyInfo.cs @@ -1,4 +1,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Net45.SODA.Utilities.Tests")] -[assembly: InternalsVisibleTo("NetCore22.SODA.Utilities.Tests")] \ No newline at end of file +[assembly: InternalsVisibleTo("NetCore22.SODA.Utilities.Tests")] +[assembly: InternalsVisibleTo("NetCore30.SODA.Utilities.Tests")] \ No newline at end of file diff --git a/appveyor.yml b/appveyor.yml index d7247cd..4a3d82b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ -version: 0.9.0-{build} +version: 0.10.1-{build} -image: Visual Studio 2017 +image: Visual Studio 2019 pull_requests: do_not_increment_build_number: true @@ -15,3 +15,6 @@ build_script: test_script: - dotnet test + +notifications: + - provider: GitHubPullRequest \ No newline at end of file diff --git a/icon.png b/icon.png new file mode 100644 index 0000000..f46a15e Binary files /dev/null and b/icon.png differ