From a32bbc1c55f9bf71520f79c011dfd4444d8c22f7 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 11:38:44 -0700 Subject: [PATCH 1/7] Create IBulkForceClient.cs --- src/ForceToolkitForNET/IBulkForceClient.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/ForceToolkitForNET/IBulkForceClient.cs diff --git a/src/ForceToolkitForNET/IBulkForceClient.cs b/src/ForceToolkitForNET/IBulkForceClient.cs new file mode 100644 index 00000000..b5843cb6 --- /dev/null +++ b/src/ForceToolkitForNET/IBulkForceClient.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using Salesforce.Common.Models.Xml; +using Salesforce.Force; + +namespace Salesforce.Force +{ + public interface IBulkForceClient + { + Task CreateUpsertJobAsync(string objectName, string externalField, BulkConstants.OperationType operationType); + Task> RunUpsertJobAndPollAsync(string objectName, string externalFieldName, BulkConstants.OperationType operationType, IEnumerable> recordsLists); + Task> RunUpsertJobAsync(string objectName, string externalFieldName, BulkConstants.OperationType operationType, IEnumerable> recordsLists); + } +} From bd85c640b9a90ba9bec079c73d7d6d7d19de162a Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 11:40:24 -0700 Subject: [PATCH 2/7] Create UpsertJobInfo.cs --- .../Models/Xml/UpsertJobInfo.cs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/CommonLibrariesForNET/Models/Xml/UpsertJobInfo.cs diff --git a/src/CommonLibrariesForNET/Models/Xml/UpsertJobInfo.cs b/src/CommonLibrariesForNET/Models/Xml/UpsertJobInfo.cs new file mode 100644 index 00000000..09fe5444 --- /dev/null +++ b/src/CommonLibrariesForNET/Models/Xml/UpsertJobInfo.cs @@ -0,0 +1,23 @@ +using System.Xml.Serialization; + +namespace Salesforce.Common.Models.Xml +{ + [XmlRoot(Namespace = "http://www.force.com/2009/06/asyncapi/dataload", + ElementName = "jobInfo", + IsNullable = false)] + public class UpsertJobInfo + { + [XmlElement(ElementName = "operation")] + public string Operation { get; set; } + + [XmlElement(ElementName = "object")] + public string Object { get; set; } + + [XmlElement(ElementName = "externalIdFieldName")] + public string ExternalField { get; set; } + + [XmlElement(ElementName = "contentType")] + public string ContentType { get; set; } + + } +} From 82b6c39be270637d830e392ebd9478aa3b05aa90 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 11:42:16 -0700 Subject: [PATCH 3/7] Change visibility of _xmlHttpClient and _jsonHttpClient from private to protected Enables derived classes to be created and implement their own job creation routines (i.e. bulk upserts). --- src/ForceToolkitForNET/ForceClient.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ForceToolkitForNET/ForceClient.cs b/src/ForceToolkitForNET/ForceClient.cs index dd7149b1..511f271a 100644 --- a/src/ForceToolkitForNET/ForceClient.cs +++ b/src/ForceToolkitForNET/ForceClient.cs @@ -13,8 +13,8 @@ namespace Salesforce.Force { public class ForceClient : IForceClient, IDisposable { - private readonly XmlHttpClient _xmlHttpClient; - private readonly JsonHttpClient _jsonHttpClient; + protected readonly XmlHttpClient _xmlHttpClient; + protected readonly JsonHttpClient _jsonHttpClient; public ForceClient(string instanceUrl, string accessToken, string apiVersion) : this(instanceUrl, accessToken, apiVersion, new HttpClient(), new HttpClient()) From ff910ac28642ac988fd1460fc499e7e36faab262 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 11:44:28 -0700 Subject: [PATCH 4/7] Create BulkForceClient.cs Derives from ForceClient (IForceClient) and implements new IBulkForceClient interface for supporting creation of bulk upsert jobs with `externalIdFieldName`. --- src/ForceToolkitForNET/BulkForceClient.cs | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 src/ForceToolkitForNET/BulkForceClient.cs diff --git a/src/ForceToolkitForNET/BulkForceClient.cs b/src/ForceToolkitForNET/BulkForceClient.cs new file mode 100644 index 00000000..9633113e --- /dev/null +++ b/src/ForceToolkitForNET/BulkForceClient.cs @@ -0,0 +1,104 @@ +using Salesforce.Common.Models.Xml; +using Salesforce.Force; +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Salesforce.Force +{ + /// + /// Extends ForceClient for creating bulk upsert jobs using new UpsertJobInfo. + /// + /// For a complete list of possible JobInfo fields, see https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/asynch_api_reference_jobinfo.htm + /// + /// + /// This class requires the following properties in ForceClient to be minimally exposed as protected: + /// - _xmlHttpClient + /// - _jsonHttpClient + /// + public class BulkForceClient : ForceClient, IDisposable, IBulkForceClient + { + public BulkForceClient(string instanceUrl, string accessToken, string apiVersion) + : this(instanceUrl, accessToken, apiVersion, new HttpClient(), new HttpClient()) + { + } + + public BulkForceClient(string instanceUrl, string accessToken, string apiVersion, HttpClient httpClientForJson, HttpClient httpClientForXml) + : base(instanceUrl, accessToken, apiVersion, httpClientForJson, httpClientForXml) + { + } + + public async Task CreateUpsertJobAsync(string objectName, string externalField, BulkConstants.OperationType operationType) + { + if (string.IsNullOrEmpty(objectName)) throw new ArgumentNullException("objectName"); + + var jobInfo = new UpsertJobInfo + { + ContentType = "XML", + Object = objectName, + ExternalField = externalField, + Operation = operationType.Value() + }; + + return await _xmlHttpClient.HttpPostAsync(jobInfo, "/services/async/{0}/job"); + } + + public async Task> RunUpsertJobAsync(string objectName, string externalFieldName, BulkConstants.OperationType operationType, + IEnumerable> recordsLists) + { + if (recordsLists == null) throw new ArgumentNullException("recordsLists"); + + var jobInfoResult = await CreateUpsertJobAsync(objectName, externalFieldName, operationType); + var batchResults = new List(); + foreach (var recordList in recordsLists) + { + batchResults.Add(await CreateJobBatchAsync(jobInfoResult, recordList)); + } + await CloseJobAsync(jobInfoResult); + return batchResults; + } + + public async Task> RunUpsertJobAndPollAsync(string objectName, string externalFieldName, BulkConstants.OperationType operationType, + IEnumerable> recordsLists) + { + const float pollingStart = 1000; + const float pollingIncrease = 2.0f; + + var batchInfoResults = await RunUpsertJobAsync(objectName, externalFieldName, operationType, recordsLists); + + var currentPoll = pollingStart; + var finishedBatchInfoResults = new List(); + while (batchInfoResults.Count > 0) + { + var removeList = new List(); + foreach (var batchInfoResult in batchInfoResults) + { + var batchInfoResultNew = await PollBatchAsync(batchInfoResult); + if (batchInfoResultNew.State.Equals(BulkConstants.BatchState.Completed.Value()) || + batchInfoResultNew.State.Equals(BulkConstants.BatchState.Failed.Value()) || + batchInfoResultNew.State.Equals(BulkConstants.BatchState.NotProcessed.Value())) + { + finishedBatchInfoResults.Add(batchInfoResultNew); + removeList.Add(batchInfoResult); + } + } + foreach (var removeItem in removeList) + { + batchInfoResults.Remove(removeItem); + } + + await Task.Delay((int)currentPoll); + currentPoll *= pollingIncrease; + } + + + var batchResults = new List(); + foreach (var batchInfoResultComplete in finishedBatchInfoResults) + { + batchResults.Add(await GetBatchResultAsync(batchInfoResultComplete)); + } + return batchResults; + } + } +} From 428b0beb575f20f70f056f64d6468a645fd42e18 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 12:06:43 -0700 Subject: [PATCH 5/7] Update IBulkForceClient.cs Derive from IForceClient --- src/ForceToolkitForNET/IBulkForceClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ForceToolkitForNET/IBulkForceClient.cs b/src/ForceToolkitForNET/IBulkForceClient.cs index b5843cb6..9e6f571d 100644 --- a/src/ForceToolkitForNET/IBulkForceClient.cs +++ b/src/ForceToolkitForNET/IBulkForceClient.cs @@ -5,7 +5,7 @@ namespace Salesforce.Force { - public interface IBulkForceClient + public interface IBulkForceClient : IForceClient { Task CreateUpsertJobAsync(string objectName, string externalField, BulkConstants.OperationType operationType); Task> RunUpsertJobAndPollAsync(string objectName, string externalFieldName, BulkConstants.OperationType operationType, IEnumerable> recordsLists); From 0aea43921899a382a00a457018f61dc7378b4739 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 12:23:37 -0700 Subject: [PATCH 6/7] Update CommonLibrariesForNET.csproj Include UpsertJobInfo.cs in project --- src/CommonLibrariesForNET/CommonLibrariesForNET.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CommonLibrariesForNET/CommonLibrariesForNET.csproj b/src/CommonLibrariesForNET/CommonLibrariesForNET.csproj index ba151d83..bbf37867 100644 --- a/src/CommonLibrariesForNET/CommonLibrariesForNET.csproj +++ b/src/CommonLibrariesForNET/CommonLibrariesForNET.csproj @@ -75,6 +75,7 @@ + @@ -105,4 +106,4 @@ --> - \ No newline at end of file + From 50ced6781cd3dc1bcafe81e94eb023dbfca6bf63 Mon Sep 17 00:00:00 2001 From: Matt Borja Date: Fri, 14 Apr 2017 12:24:16 -0700 Subject: [PATCH 7/7] Update ForceToolkitForNET.csproj Include IBulkForceClient.cs and BulkForceClient.cs in project --- src/ForceToolkitForNET/ForceToolkitForNET.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ForceToolkitForNET/ForceToolkitForNET.csproj b/src/ForceToolkitForNET/ForceToolkitForNET.csproj index 9592b245..6cae4f4b 100644 --- a/src/ForceToolkitForNET/ForceToolkitForNET.csproj +++ b/src/ForceToolkitForNET/ForceToolkitForNET.csproj @@ -49,6 +49,8 @@ + +