Merge pull request #165 from CSCfi/CSCTTV-3986-3987-3988-3994-3995-39…

…98-search-after-endpoints

CSCTTV 3986 3987 3988 3994 3995 3998 export endpoints

aspnetcore/src/ApiModels/Query/GetFundingCallQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching funding calls.
 /// </summary>
 /// <see cref="FundingCall"/>
-public class GetFundingCallQueryParameters : PaginationQueryParameters
+public class GetFundingCallQueryParameters
 {
     /// <summary>
     /// One of the fields nameFi, nameSV, nameEn contains the full text.

aspnetcore/src/ApiModels/Query/GetFundingDecisionQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching funding decisions.
 /// </summary>
 /// <see cref="FundingDecision"/>
-public class GetFundingDecisionQueryParameters : PaginationQueryParameters
+public class GetFundingDecisionQueryParameters
 {
     /// <summary>
     /// One of the fields nameFi, nameSV, nameEn contains the full text.

aspnetcore/src/ApiModels/Query/GetInfrastructuresQueryParameters.cs

@@ -3,7 +3,7 @@
 /// <summary>
 /// Hakuparametrit infrastruktuurien hakemiseen.
 /// </summary>
-public class GetInfrastructuresQueryParameters : PaginationQueryParameters
+public class GetInfrastructuresQueryParameters
 {
     /// <summary>
     /// Infrastruktuurin nimi.

aspnetcore/src/ApiModels/Query/GetPublicationsQueryParameters.cs

@@ -6,7 +6,7 @@ namespace ResearchFi.Query;
 /// Query parameters for searching publications.
 /// </summary>
 /// <see cref="Publication"/>
-public class GetPublicationsQueryParameters : PaginationQueryParameters
+public class GetPublicationsQueryParameters
 {
     /// <summary>
     /// The field name contains text.

aspnetcore/src/ApiModels/Query/PaginationQueryParameters.cs

@@ -19,7 +19,7 @@ public int PageNumber
     }
 
     /// <summary>
-    /// Number of results on page. Optional. Default value 20. Maximum permissible value 100. Maximum possible result set of 10,000 results.
+    /// Number of results on page. Optional. Default value 20. Maximum permissible value 100. Maximum possible result set of 10000 results.
     /// </summary>
     public int PageSize
     {

aspnetcore/src/ApiModels/Query/SearchAfterQueryParameters.cs

@@ -0,0 +1,30 @@
+namespace ResearchFi.Query;
+
+/// <summary>
+/// Vientiin liittyvät tiedot.
+/// </summary>
+public class SearchAfterQueryParameters
+{
+    private const int DefaultPageSize = 50;
+    private const int MaximumPageSize = 1000;
+    private int _pageSize = DefaultPageSize;
+    private long? _nextPageToken = null;
+
+    /// <summary>
+    /// Number of results on page. Optional. Default value 50. Maximum permissible value 1000.
+    /// </summary>
+    public int PageSize
+    {
+        get => _pageSize;
+        set => _pageSize = value < 1 ? DefaultPageSize : (value > MaximumPageSize ? MaximumPageSize : value);
+    }
+
+    /// <summary>
+    /// Value from previous query response header "x-next-page-token". Leave empty in the first query.
+    /// </summary>
+    public long? NextPageToken
+    {
+        get => _nextPageToken;
+        set => _nextPageToken = value;
+    }
+}

aspnetcore/src/ElasticService/ElasticSearchQueryGenerators/IQueryGenerator.cs

@@ -5,5 +5,6 @@ namespace CSC.PublicApi.ElasticService.ElasticSearchQueryGenerators;
 public interface IQueryGenerator<TIn, TOut> where TOut : class
 {
     Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuery(TIn searchParameters, int pageNumber, int pageSize);
+    Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuerySearchAfter(TIn searchParameters, int pageSize, long? searchAfter);
     Func<SearchDescriptor<TOut>, ISearchRequest> GenerateSingleQuery(string id);
 }

aspnetcore/src/ElasticService/ElasticSearchQueryGenerators/QueryGeneratorBase.cs

@@ -22,6 +22,27 @@ public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuery(TIn searchPara
             .Query(GenerateQueryForSearch(searchParameters));
     }
 
+    public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateQuerySearchAfter(TIn searchParameters, int pageSize, long? searchAfter)
+    {
+        var indexName = _configuration.GetIndexNameForType(typeof(TOut));
+
+        if (searchAfter == null) {
+            return descriptor => descriptor
+                .Index(indexName)
+                .Take(pageSize)
+                .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
+                .Query(GenerateQueryForSearch(searchParameters));
+        }
+        else {
+            return descriptor => descriptor
+                .Index(indexName)
+                .Take(pageSize)
+                .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
+                .Query(GenerateQueryForSearch(searchParameters))
+                .SearchAfter(searchAfter);
+        }
+    }
+
     public Func<SearchDescriptor<TOut>, ISearchRequest> GenerateSingleQuery(string id)
     {
         var indexName = _configuration.GetIndexNameForType(typeof(TOut));

aspnetcore/src/ElasticService/ElasticSearchService.cs

@@ -32,6 +32,27 @@ public ElasticSearchService(
         return (searchResult.Documents, new SearchResult(pageNumber, pageSize, searchResult.HitsMetadata?.Total.Value));
     }
 
+    public async Task<(IEnumerable<TOut>, SearchAfterResult)> SearchAfter(TIn parameters, int pageSize, long? searchAfter)
+    {
+        var query = _queryGenerator.GenerateQuerySearchAfter(parameters, pageSize, searchAfter);
+
+        var searchResult = await _elasticClient.SearchAsync(query);
+
+        if (Debugger.IsAttached)
+        {
+            // Enables seeing the query sent to elastic and the response in the log when debugging.
+            Console.WriteLine(searchResult.DebugInformation);            
+        }
+
+        long? searchAfterValue = null;
+
+        if (searchResult.Hits.Count > 0) {
+            searchAfterValue = (long)searchResult.Hits.Last().Sorts.First();
+        }
+
+        return (searchResult.Documents, new SearchAfterResult(searchAfterValue, pageSize));
+    }
+
     public async Task<TOut?> GetSingle(string id)
     {
         var query = _queryGenerator.GenerateSingleQuery(id);

aspnetcore/src/ElasticService/ISearchService.cs

@@ -3,6 +3,6 @@
 public interface ISearchService<TIn, TOut> where TOut : class
 {
     Task<(IEnumerable<TOut>, SearchResult)> Search(TIn searchParameters, int pageNumber, int pageSize);
-
+    Task<(IEnumerable<TOut>, SearchAfterResult)> SearchAfter(TIn searchParameters, int pageSize, long? searchAfter);
     Task<TOut?> GetSingle(string id);
 }

aspnetcore/src/ElasticService/SearchAfterResult.cs

@@ -0,0 +1,12 @@
+namespace CSC.PublicApi.ElasticService;
+
+public class SearchAfterResult
+{
+    public long? SearchAfter { get; }
+    public int PageSize { get; }
+    public SearchAfterResult(long? searchAfter, int pageSize)
+    {
+        SearchAfter = searchAfter;
+        PageSize = pageSize;
+    }
+}

aspnetcore/src/Interface/Configuration/SwaggerExtensions.cs

@@ -76,6 +76,7 @@ public static void UseSwaggerAndSwaggerUI(this WebApplication app)
             foreach (var description in app.Services.GetRequiredService<IApiVersionDescriptionProvider>().ApiVersionDescriptions)
             {
                 options.SwaggerEndpoint($"{description.GroupName}/swagger.json", description.GroupName.ToUpperInvariant());
+                options.ConfigObject.AdditionalItems.Add("syntaxHighlight", false); // disable to improve performance with large responses
             }
         });
     }

aspnetcore/src/Interface/Controllers/FundingCallController.cs

@@ -30,7 +30,7 @@ public FundingCallController(
     /// <summary>
     /// Endpoint for filtering funding calls using the specified query parameters.
     /// </summary>
-    /// <param name="queryParameters">The query parameters for filtering the results.</param>
+    /// <param name="fundingCallQueryParameters">The query parameters for filtering the results.</param>
     /// <returns>Paged search result as a collection of <see cref="FundingCall"/> objects.</returns>
     /// <response code="200">Ok.</response>
     /// <response code="401">Unauthorized.</response>
@@ -43,9 +43,9 @@ public FundingCallController(
     [ProducesResponseType(typeof(IEnumerable<FundingCall>), StatusCodes.Status200OK)]
     [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
     [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
-    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters queryParameters)
+    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters fundingCallQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (fundingCalls, searchResult) = await _service.GetFundingCalls(queryParameters);
+        var (fundingCalls, searchResult) = await _service.GetFundingCalls(fundingCallQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);
 

aspnetcore/src/Interface/Controllers/FundingCallExportController.cs

@@ -0,0 +1,54 @@
+using CSC.PublicApi.Interface.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using ResearchFi.FundingCall;
+using ResearchFi.Query;
+using Serilog;
+
+namespace CSC.PublicApi.Interface.Controllers;
+
+[ApiController]
+[ApiVersion(ApiConstants.ApiVersion1)]
+[Route("v{version:apiVersion}/funding-calls-export")]
+public class FundingCallExportController : ControllerBase
+{
+    private readonly ILogger<FundingCallExportController> _logger;
+    private readonly IFundingCallService _service;
+    private readonly IDiagnosticContext _diagnosticContext;
+
+    public FundingCallExportController(
+        ILogger<FundingCallExportController> logger,
+        IFundingCallService service,
+        IDiagnosticContext diagnosticContext)
+    {
+        _logger = logger;
+        _service = service;
+        _diagnosticContext = diagnosticContext;
+        _diagnosticContext.Set(ApiConstants.LogResourceType_PropertyName, ApiConstants.LogResourceType_FundingCall);
+    }
+
+    /// <summary>
+    /// Endpoint for bypassing the limit of 10000 records for funding calls.
+    /// </summary>
+    /// <param name="fundingCallQueryParameters">The query parameters for filtering the results.</param>
+    /// <returns>Paged search result as a collection of <see cref="FundingCall"/> objects.</returns>
+    /// <response code="200">Ok.</response>
+    /// <response code="401">Unauthorized.</response>
+    /// <response code="403">Forbidden.</response>
+    [HttpGet(Name = "GetFundingCallExport")]
+    [MapToApiVersion(ApiConstants.ApiVersion1)]
+    [Authorize(Policy = ApiPolicies.FundingCall.Read)]
+    [Produces(ApiConstants.ContentTypeJson)]
+    [Consumes(ApiConstants.ContentTypeJson)]
+    [ProducesResponseType(typeof(IEnumerable<FundingCall>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
+    public async Task<IEnumerable<FundingCall>> Get([FromQuery] GetFundingCallQueryParameters fundingCallQueryParameters, [FromQuery] SearchAfterQueryParameters searchAfterQueryParameters)
+    {
+        var (fundingCalls, searchAfterResult) = await _service.GetFundingCallsSearchAfter(fundingCallQueryParameters, searchAfterQueryParameters);
+
+        ResponseHelper.AddPaginationResponseHeadersSearchAfter(HttpContext, searchAfterResult);
+
+        return fundingCalls;
+    }
+}

aspnetcore/src/Interface/Controllers/FundingDecisionController.cs

@@ -32,6 +32,7 @@ public FundingDecisionController(
     /// <summary>
     /// Endpoint for filtering funding decisions using the specified query parameters.
     /// </summary>
+    /// <param name="fundingDecisionQueryParameters">The query parameters for filtering the results.</param>
     /// <returns>Paged search result as a collection of <see cref="FundingDecision"/> objects.</returns>
     /// <response code="200">Ok.</response>
     /// <response code="401">Unauthorized.</response>
@@ -44,9 +45,9 @@ public FundingDecisionController(
     [ProducesResponseType(typeof(IEnumerable<FundingDecision>), StatusCodes.Status200OK)]
     [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
     [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
-    public async Task<IEnumerable<FundingDecision>> Get([FromQuery] GetFundingDecisionQueryParameters queryParameters)
+    public async Task<IEnumerable<FundingDecision>> Get([FromQuery] GetFundingDecisionQueryParameters fundingDecisionQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (fundingDecisions, searchResult) = await _service.GetFundingDecisions(queryParameters);
+        var (fundingDecisions, searchResult) = await _service.GetFundingDecisions(fundingDecisionQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);
 

aspnetcore/src/Interface/Controllers/FundingDecisionExportController.cs

@@ -0,0 +1,56 @@
+using CSC.PublicApi.Interface.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using ResearchFi.FundingDecision;
+using ResearchFi.Query;
+using Serilog;
+
+namespace CSC.PublicApi.Interface.Controllers;
+
+[ApiController]
+[ApiVersion(ApiVersion)]
+[Route("v{version:apiVersion}/funding-decisions-export")]
+
+public class FundingDecisionExportController : ControllerBase
+{
+    private const string ApiVersion = "1.0";
+    private readonly ILogger<FundingDecisionExportController> _logger;
+    private readonly IFundingDecisionService _service;
+    private readonly IDiagnosticContext _diagnosticContext;
+
+    public FundingDecisionExportController(
+        ILogger<FundingDecisionExportController> logger,
+        IFundingDecisionService service,
+        IDiagnosticContext diagnosticContext)
+    {
+        _logger = logger;
+        _service = service;
+        _diagnosticContext = diagnosticContext;
+        _diagnosticContext.Set(ApiConstants.LogResourceType_PropertyName, ApiConstants.LogResourceType_FundingDecision);
+    }
+
+    /// <summary>
+    /// Endpoint for bypassing the limit of 10000 records for funding decisions.
+    /// </summary>
+    /// <param name="fundingDecisionQueryParameters">The query parameters for filtering the results.</param>
+    /// <returns>Paged search result as a collection of <see cref="FundingDecision"/> objects.</returns>
+    /// <response code="200">Ok.</response>
+    /// <response code="401">Unauthorized.</response>
+    /// <response code="403">Forbidden.</response>
+    [HttpGet(Name = "GetFundingDecisionExport")]
+    [MapToApiVersion(ApiVersion)]
+    [Authorize(Policy = ApiPolicies.FundingDecision.Read)]
+    [Produces(ApiConstants.ContentTypeJson)]
+    [Consumes(ApiConstants.ContentTypeJson)]
+    [ProducesResponseType(typeof(IEnumerable<FundingDecision>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
+    public async Task<IEnumerable<FundingDecision>> Get([FromQuery] GetFundingDecisionQueryParameters fundingDecisionQueryParameters, [FromQuery] SearchAfterQueryParameters searchAfterQueryParameters)
+    {
+        var (fundingDecisions, searchAfterResult) = await _service.GetFundingDecisionsSearchAfter(fundingDecisionQueryParameters, searchAfterQueryParameters);
+
+        ResponseHelper.AddPaginationResponseHeadersSearchAfter(HttpContext, searchAfterResult);
+
+        return fundingDecisions;
+    }
+}

aspnetcore/src/Interface/Controllers/InfrastructureController.cs

@@ -32,14 +32,22 @@ public InfrastructureController(
     /// <summary>
     /// Search Infrastructures
     /// </summary>
-    /// <param name="queryParameters">Query parameters for filtering the results.</param>
-    /// <returns></returns>
+    /// <param name="infrastructuresQueryParameters">The query parameters for filtering the results.</param>
+    /// <returns>Paged search result as a collection of <see cref="Infrastructure"/> objects.</returns>
+    /// <response code="200">Ok.</response>
+    /// <response code="401">Unauthorized.</response>
+    /// <response code="403">Forbidden.</response>
     [HttpGet(Name = "GetInfrastructure")]
     [MapToApiVersion(ApiVersion)]
     [Authorize(Policy = ApiPolicies.Infrastructure.Read)]
-    public async Task<IEnumerable<Infrastructure>> Get([FromQuery] GetInfrastructuresQueryParameters queryParameters)
+    [Produces(ApiConstants.ContentTypeJson)]
+    [Consumes(ApiConstants.ContentTypeJson)]
+    [ProducesResponseType(typeof(IEnumerable<Infrastructure>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
+    public async Task<IEnumerable<Infrastructure>> Get([FromQuery] GetInfrastructuresQueryParameters infrastructuresQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (infrastructures, searchResult) = await _service.GetInfrastructures(queryParameters);
+        var (infrastructures, searchResult) = await _service.GetInfrastructures(infrastructuresQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);
 

aspnetcore/src/Interface/Controllers/OrganizationController.cs

@@ -32,14 +32,22 @@ public OrganizationController(
     /// <summary>
     /// Hae organisaatioita
     /// </summary>
-    /// <param name="queryParameters"></param>
-    /// <returns></returns>
+    /// <param name="organizationsQueryParameters">The query parameters for filtering the results.</param>
+    /// <returns>Paged search result as a collection of <see cref="Organization"/> objects.</returns>
+    /// <response code="200">Ok.</response>
+    /// <response code="401">Unauthorized.</response>
+    /// <response code="403">Forbidden.</response>
     [HttpGet(Name = "GetOrganization")]
     [MapToApiVersion(ApiVersion)]
     [Authorize(Policy = ApiPolicies.Organization.Read)]
-    public async Task<IEnumerable<Organization>> Get([FromQuery] GetOrganizationsQueryParameters queryParameters)
+    [Produces(ApiConstants.ContentTypeJson)]
+    [Consumes(ApiConstants.ContentTypeJson)]
+    [ProducesResponseType(typeof(IEnumerable<Organization>), StatusCodes.Status200OK)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void),StatusCodes.Status403Forbidden)]
+    public async Task<IEnumerable<Organization>> Get([FromQuery] GetOrganizationsQueryParameters organizationsQueryParameters, [FromQuery] PaginationQueryParameters paginationQueryParameters)
     {
-        var (organizations, searchResult) = await _service.GetOrganizations(queryParameters);
+        var (organizations, searchResult) = await _service.GetOrganizations(organizationsQueryParameters, paginationQueryParameters);
 
         ResponseHelper.AddPaginationResponseHeaders(HttpContext, searchResult);