From 6b9204c4db7c225af1dc2a664eddedf0bd62f6b4 Mon Sep 17 00:00:00 2001 From: Oscar Martinez Valero Date: Fri, 26 Apr 2024 17:31:26 +0200 Subject: [PATCH] Add SharePoint List Paginated function --- .../SharePoint/{GetList.pq => GetSPList.pq} | 179 ++++++++++-------- 1 file changed, 102 insertions(+), 77 deletions(-) rename Functions/SharePoint/{GetList.pq => GetSPList.pq} (61%) diff --git a/Functions/SharePoint/GetList.pq b/Functions/SharePoint/GetSPList.pq similarity index 61% rename from Functions/SharePoint/GetList.pq rename to Functions/SharePoint/GetSPList.pq index 50b70db..edf5468 100644 --- a/Functions/SharePoint/GetList.pq +++ b/Functions/SharePoint/GetSPList.pq @@ -1,77 +1,102 @@ -//Function documentation template by Oscar Martinez https://bibb.pro - -let - // Define metadata for the function, describing its purpose and usage. - metaDocumentation = type function ( - baseURL as (type text meta [ - Documentation.FieldCaption = "SharePoint base url without ending ""/""", - Documentation.SampleValues = {"https://"}, - Formatting.IsMultiLine = false, - Formatting.IsCode = false - ]), - listName as (type text meta [ - Documentation.FieldCaption = "SharePoint list or Document Library", - Documentation.SampleValues = {""}, - Formatting.IsMultiLine = false, - Formatting.IsCode = false - ]), - waitTime as (type number meta [ - Documentation.FieldCaption = "Wait time between API calls in seconds", - Documentation.SampleValues = {0.1}, - Formatting.IsMultiLine = false, - Formatting.IsCode = false - ]) - ) as any meta [ - Documentation.Name = "SharePoint List Paginated", - Documentation.LongDescription = - //This is the description of the documentation, it only accepts a handful of HTML tags for formatting. - " -

SharePoint List Paginated

-
  • Creator: oscar Martinez.
    • -
  • LinkedIn: https://www.linkedin.com/in/oscarmartinezv/
    • -
  • Web: https://bibb.pro
    • - -

    This function will retrieve all the elements of a SharePoint list or document library. - It works with very big pages as it has paging mechanism

    -

    The function takes three parameters:

    -
      -
    • baseURL: SharePoint base url without ending ""/"".
      • -
    • listName: SharePoint list or Document Library.
      • -
    • waitTime: Wait time between API calls in seconds.
      • -
    -

    References

    -

    Big thanks to Rob Reily and googlogmobi for their entries that made this possible.

    -

    https://www.linkedin.com/pulse/loading-data-paged-related-from-ms-graph-api-power-bi-rob-reilly/

    -

    https://community.powerbi.com/t5/Power-Query/Dynamic-data-sources-aren-t-refreshed-in-the-Power-BI-service/td-p/1563630

    - " - ], - // Define the main function - myFunction = (baseURL as text, listName as text, waitTime as number)=> - let - GetPages = (Path) => - let - Source = Json.Document( - Web.Contents( - baseURL, - [ - RelativePath = Path, - Headers=[ - Accept="application/json;odata=nometadata" - ] - ] - ) - ), - ll= Source[value], - Next = Text.Replace(Source[odata.nextLink], baseURL, ""), - result = try @ll & Function.InvokeAfter(()=> @GetPages(Next) , #duration(0,0,0,waitTime)) otherwise @ll - in - result, - - Fullset = GetPages("/_api/web/lists/getbytitle('"&listName&"')/items?" - ), - #"Converted to Table" = Table.FromList(Fullset, Splitter.SplitByNothing(), null, null, ExtraValues.Error) - in - #"Converted to Table" -in - // Apply the function metadata to myFunction. - Value.ReplaceType(myFunction, metaDocumentation) +let + // Define metadata for the function, describing its purpose and usage. + metaDocumentation = type function ( + baseURL as (type text meta [ + Documentation.FieldCaption = "SharePoint base url without ending ""/""", + Documentation.SampleValues = {"https://"}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]), + listName as (type text meta [ + Documentation.FieldCaption = "SharePoint list or Document Library", + Documentation.SampleValues = {""}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]), + optional waitTime as (type number meta [ + Documentation.FieldCaption = "Wait time between API calls in seconds", + Documentation.SampleValues = {0.1}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]), + optional selectedColumns as (type text meta [ + Documentation.FieldCaption = "Selected columns separated by comma", + Documentation.SampleValues = {"Id,FsObjType,Person/EMail"}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]), + optional expandedColumns as (type text meta [ + Documentation.FieldCaption = "Selected the columns that require expansion", + Documentation.SampleValues = {"Person"}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]), + optional appliedFilter as (type text meta [ + Documentation.FieldCaption = "Filter to be applied", + Documentation.SampleValues = {"FSObjType eq 0"}, + Formatting.IsMultiLine = false, + Formatting.IsCode = false + ]) + ) as any meta [ + Documentation.Name = "SharePoint List Paginated", + Documentation.LongDescription = + //This is the description of the documentation, it only accepts a handful of HTML tags for formatting. + " +

    SharePoint List Paginated

    +
  • Creator: oscar Martinez.
    • +
  • LinkedIn: https://www.linkedin.com/in/oscarmartinezv/
    • +
  • Web: https://bibb.pro
    • + +

    This function will retrieve all the elements of a SharePoint list or document library. + It works with very big document libraries or lists as it has paging mechanism

    + +

    References

    +

    Big thanks to Rob Reily and googlogmobi for their entries that made this possible.

    +

    https://www.linkedin.com/pulse/loading-data-paged-related-from-ms-graph-api-power-bi-rob-reilly/

    +

    https://community.powerbi.com/t5/Power-Query/Dynamic-data-sources-aren-t-refreshed-in-the-Power-BI-service/td-p/1563630

    + " + ], + // Define the main function + myFunction = ( + baseURL as text, + listName as text, + optional waitTime as number, + optional selectedColumns as text, + optional expandedColumns as text, + optional appliedFilter as text + )=> + let + selectSPColumns= if selectedColumns is null then "" else "&$select=" & selectedColumns, + selectExpandedColumns= if expandedColumns is null then "" else "&$expand=" & expandedColumns, + selectedWaitTime = if waitTime is null then 0.1 else waitTime, + selectedAppliedFilter = if appliedFilter is null then "" else "&$filter=" & appliedFilter, + GetPages = (Path) => + let + Source = Json.Document( + Web.Contents( + baseURL, + [ + RelativePath = Path, + Headers=[ + Accept="application/json;odata=nometadata" + ] + ] + ) + ), + ll= Source[value], + Next = Text.Replace(Source[odata.nextLink], baseURL, ""), + result = try @ll & Function.InvokeAfter(()=> @GetPages(Next) , #duration(0,0,0,selectedWaitTime)) otherwise @ll + in + result, + + Fullset = GetPages("/_api/web/lists/getbytitle('"&listName&"')/items?" + & selectSPColumns + & selectExpandedColumns + & selectedAppliedFilter + ), + #"Converted to Table" = Table.FromList(Fullset, Splitter.SplitByNothing(), null, null, ExtraValues.Error) + in + #"Converted to Table" +in + // Apply the function metadata to myFunction. + Value.ReplaceType(myFunction, metaDocumentation) \ No newline at end of file