From 9b8da9d6f12c55d8379c6121c8b1fdaceffef8ad Mon Sep 17 00:00:00 2001 From: Peter Vigier <3204560+p3t@users.noreply.github.com> Date: Thu, 10 Oct 2024 11:12:16 +0200 Subject: [PATCH] Feature/small api improvements (#51) * support paths directly in `Filter` * Glitch in generics of PageRequest * support totalCount calculation for a request (on demand) The total is passed via the request(s)/cursor back to the client, so it should be calculated only once and not for each request again. --- README.md | 23 +++++- .../cursorpaging/jpa/api/DtoPageRequest.java | 4 +- .../jpa/serializer/FromDtoMapper.java | 2 + .../jpa/serializer/ToDtoMapper.java | 9 ++- .../src/main/proto/pagerequest.proto | 1 + .../jpa/serializer/SerializerTest.java | 12 ++++ .../io/vigier/cursorpaging/jpa/Filter.java | 12 ++++ .../java/io/vigier/cursorpaging/jpa/Page.java | 4 ++ .../vigier/cursorpaging/jpa/PageRequest.java | 72 ++++++++++++++++--- .../jpa/impl/CursorPageRepositoryImpl.java | 9 ++- .../jpa/itest/PostgreSqlCursorPageTest.java | 26 ++++++- 11 files changed, 154 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9f129e6..229ea50 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ public void queryData() { } ``` -### There is no total count in the Page... +### There is no total count in the Page - by default Executing a count operation can be a quite expensive operation! Therefore, the total count of records is not part of the page response. It is also usually not required to re-count all records with each page request! So, if you need to know the total count of records, you can execute a count query on the repository: @@ -249,6 +249,27 @@ public long queryCount( PageRequest request ) { The request (if available) should be passed as it might contain filters which reduce returned count. +It is also possible to instruct the repository to do the count for the page request. Still, the total-count will be stored within the returned self & next-page requests, and will not be recalculated for the subsequent requests! + +```java +public void queryData() { + final PageRequest request = PageRequest.create( + b -> b.pageSize( 5 ).withTotalCount( true ).asc( DataRecord_.id ) ); + + final var page = dataRecordRepository.loadPage( request ); + log( "Total records: " + page.totalCount() ); + + // insert some data... + + final var page2 = dataRecordRepository.loadPage( page.next().orElseThrow() ); + log( "Total records: " + page2.totalCount() == page.totalCount() ); // will output true + + // force recalculating the total count + final var page3 = dataRecordRepository.loadPage( secondPage.next().orElseThrow().withEnableTotalCount( true ) ); + log( "Total records: " + page3.totalCount() ); // new total count +} +``` + ### Example: Extend filter with custom rules In some cases you might want to filter the results by defining your own custom rules. One example where this could be useful is, when you need to implement sophisticated access rights (ACLs). diff --git a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/api/DtoPageRequest.java b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/api/DtoPageRequest.java index f32c863..bf7abd7 100644 --- a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/api/DtoPageRequest.java +++ b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/api/DtoPageRequest.java @@ -26,6 +26,8 @@ public class DtoPageRequest { @Max( 100 ) private int pageSize = 100; // Must not be final! + private boolean withTotalCount; + public DtoPageRequest withPageSize( final int pageSize ) { this.pageSize = pageSize; return this; @@ -54,7 +56,7 @@ public PageRequest toPageRequest( final Function attri final Attribute attribute = attributeProvider.apply( name ); b.filter( Filter.create( fb -> fb.attribute( attribute ).values( values ) ) ); } ); - b.pageSize( pageSize ); + b.pageSize( pageSize ).enableTotalCount( withTotalCount ); } ); } diff --git a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/FromDtoMapper.java b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/FromDtoMapper.java index 0040cc6..1bef851 100644 --- a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/FromDtoMapper.java +++ b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/FromDtoMapper.java @@ -32,6 +32,8 @@ public static FromDtoMapper create( final Consumer map() { return PageRequest.builder().positions( positions() ).filters( filters() ).pageSize( request.getPageSize() ) + .enableTotalCount( request.hasTotalCount() ) + .totalCount( request.hasTotalCount() ? request.getTotalCount() : null ) .build(); } diff --git a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/ToDtoMapper.java b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/ToDtoMapper.java index bc1d543..dd6b92a 100644 --- a/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/ToDtoMapper.java +++ b/cursorpaging-jpa-api/src/main/java/io/vigier/cursorpaging/jpa/serializer/ToDtoMapper.java @@ -12,9 +12,12 @@ class ToDtoMapper { private final PageRequest pageRequest; public Cursor.PageRequest map() { - return Cursor.PageRequest.newBuilder().addAllPositions( positions() ).setPageSize( pageRequest.pageSize() ) - .addAllFilters( filters() ) - .build(); + final var builder = Cursor.PageRequest.newBuilder() + .addAllPositions( positions() ) + .setPageSize( pageRequest.pageSize() ) + .addAllFilters( filters() ); + pageRequest.totalCount().ifPresent( builder::setTotalCount ); + return builder.build(); } diff --git a/cursorpaging-jpa-api/src/main/proto/pagerequest.proto b/cursorpaging-jpa-api/src/main/proto/pagerequest.proto index eb1b4a0..fcf193f 100644 --- a/cursorpaging-jpa-api/src/main/proto/pagerequest.proto +++ b/cursorpaging-jpa-api/src/main/proto/pagerequest.proto @@ -31,4 +31,5 @@ message PageRequest { int32 page_size = 1; repeated Position positions = 2; repeated Filter filters = 3; + optional int64 total_count = 4; } \ No newline at end of file diff --git a/cursorpaging-jpa-api/src/test/java/io/vigier/cursorpaging/jpa/serializer/SerializerTest.java b/cursorpaging-jpa-api/src/test/java/io/vigier/cursorpaging/jpa/serializer/SerializerTest.java index c233b60..e3fe386 100644 --- a/cursorpaging-jpa-api/src/test/java/io/vigier/cursorpaging/jpa/serializer/SerializerTest.java +++ b/cursorpaging-jpa-api/src/test/java/io/vigier/cursorpaging/jpa/serializer/SerializerTest.java @@ -72,6 +72,18 @@ void shouldSerializeReversedPageRequests() { } + @Test + void shouldSerializeTotalCountIfPresent() { + final var request = createPageRequest().copy( b -> b.enableTotalCount( true ).totalCount( 42L ) ); + final EntitySerializer serializer = EntitySerializer.create(); + final var serializedRequest = serializer.toBase64( request ); + final var deserializedRequest = serializer.toPageRequest( serializedRequest ); + assertThat( deserializedRequest ).isEqualTo( request ).satisfies( r -> { + assertThat( r.totalCount() ).isPresent().get().isEqualTo( 42L ); + assertThat( r.enableTotalCount() ).isTrue(); + } ); + } + @Test void shouldLearnAttributesBySerializing() { final var request = createPageRequest(); diff --git a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Filter.java b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Filter.java index 179e280..e8c76e5 100644 --- a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Filter.java +++ b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Filter.java @@ -55,6 +55,18 @@ public FilterBuilder attribute( final SingularAttribute>... attributes ) { + this.attribute = Attribute.path( attributes ); + return this; + } + public FilterBuilder attribute( final Attribute attribute ) { this.attribute = attribute; return this; diff --git a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Page.java b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Page.java index 72f631e..e0c7355 100644 --- a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Page.java +++ b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/Page.java @@ -80,6 +80,10 @@ public List getContent() { return Collections.unmodifiableList( content ); } + public Optional getTotalCount() { + return self.totalCount(); + } + /** * The request which can be used to fetching the next page. * diff --git a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/PageRequest.java b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/PageRequest.java index db3e18a..1450000 100644 --- a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/PageRequest.java +++ b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/PageRequest.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -25,7 +26,7 @@ * * @param the entity type */ -@Builder( toBuilder = true ) +@Builder @Getter @Accessors( fluent = true ) @EqualsAndHashCode @@ -64,6 +65,13 @@ public class PageRequest { @Builder.Default private final int pageSize = DEFAULT_PAGE_SIZE; + /** + * Control if the total element count should be calculated if missing in the request + */ + private final boolean enableTotalCount; + + private final Long totalCount; + /** * Adding some short-cut builder methods to create a request * @@ -78,7 +86,7 @@ public static class PageRequestBuilder { * @return the builder */ public PageRequestBuilder firstPage( final Order order, - final SingularAttribute> attribute ) { + final SingularAttribute> attribute ) { return addPosition( Position.create( b -> b.attribute( Attribute.of( attribute ) ).order( order ) ) ); } @@ -88,7 +96,7 @@ public PageRequestBuilder firstPage( final Order order, * @param attribute the attribute used to create a position (ascending ordered) * @return the builder */ - public PageRequestBuilder asc( final SingularAttribute> attribute ) { + public PageRequestBuilder asc( final SingularAttribute> attribute ) { return firstPage( Order.ASC, attribute ); } @@ -98,7 +106,7 @@ public PageRequestBuilder asc( final SingularAttribute desc( final SingularAttribute> attribute ) { + public PageRequestBuilder desc( final SingularAttribute> attribute ) { return firstPage( Order.DESC, attribute ); } @@ -154,6 +162,7 @@ public PageRequestBuilder filters( final Collection filters ) { return this; } + private PageRequestBuilder addPosition( final Position pos ) { if ( this.positions == null ) { this.positions = new ArrayList<>( 3 ); @@ -176,6 +185,49 @@ public static PageRequest create( final Consumer> c return builder.build(); } + /** + * Create a new page-request from the current one, but with the provided customizer applied + * + * @param c customizer for the copy + * @return A new page-request with existing and customized attributes + */ + public PageRequest copy( Consumer> c ) { + PageRequestBuilder builder = PageRequest.builder() + .totalCount( totalCount ) + .enableTotalCount( enableTotalCount ) + .pageSize( pageSize ); + c.accept( builder ); + if ( !builder.filters$set && !filters.isEmpty() ) { + builder.filters( filters ); + } + if ( (builder.rules == null || builder.rules.isEmpty()) && !rules.isEmpty() ) { + builder.rules( rules ); + } + if ( (builder.positions == null || builder.positions.isEmpty()) && !positions.isEmpty() ) { + builder.positions( positions ); + } + return builder.build(); + } + + /** + * Enable the total count calculation for the request.
Setting {@code enable = true} forces also the + * re-calculation of the total count for a page-request where the total-count is already present. + * + * @return A copy of the page-request where the total-count is removed and the enable flag is set accordingly + */ + public PageRequest withEnableTotalCount( boolean enable ) { + return copy( b -> b.enableTotalCount( enable ).totalCount( null ) ); + } + + /** + * Get the total count if present + * + * @return the total count if present + */ + public Optional totalCount() { + return Optional.ofNullable( totalCount ); + } + /** * Create a new {@linkplain PageRequest} pointing to the position defined through the attributes of the provided * entity. @@ -184,16 +236,14 @@ public static PageRequest create( final Consumer> c * @return A new PageRequest with the positions set to the values of the provided entity */ public PageRequest positionOf( final E entity ) { - return create( b -> b.positions( positions.stream() - .map( p -> p.positionOf( entity ) ) - .toList() ).pageSize( this.pageSize ) ); + return create( b -> b.positions( positions.stream().map( p -> p.positionOf( entity ) ).toList() ) + .pageSize( this.pageSize ) + .totalCount( this.totalCount ) + .enableTotalCount( this.enableTotalCount ) ); } public PageRequest toReversed() { - return PageRequest.builder() - .rules( rules ).filters( filters ).positions( positions.stream().map( Position::toReversed ).toList() ) - .pageSize( pageSize ) - .build(); + return copy( b -> b.positions( positions.stream().map( Position::toReversed ).toList() ) ); } public boolean isFirstPage() { diff --git a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/impl/CursorPageRepositoryImpl.java b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/impl/CursorPageRepositoryImpl.java index 6f34005..cfaec5a 100644 --- a/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/impl/CursorPageRepositoryImpl.java +++ b/cursorpaging-jpa/src/main/java/io/vigier/cursorpaging/jpa/impl/CursorPageRepositoryImpl.java @@ -66,9 +66,12 @@ public Page loadPage( final PageRequest request ) { .setMaxResults( getMaxResultSize( request ) ) .getResultList(); - return Page.create( b -> b.content( toContent( results, request ) ) - .self( request ) - .next( toNextRequest( results, request ) ) ); + final PageRequest self = request.enableTotalCount() && request.totalCount().isEmpty() ? request.copy( + b -> b.totalCount( count( request ) ) ) : request; + + return Page.create( b -> b.content( toContent( results, self ) ) // + .self( self ) // + .next( toNextRequest( results, self ) ) ); } @Override diff --git a/cursorpaging-jpa/src/test/java/io/vigier/cursorpaging/jpa/itest/PostgreSqlCursorPageTest.java b/cursorpaging-jpa/src/test/java/io/vigier/cursorpaging/jpa/itest/PostgreSqlCursorPageTest.java index a04cb56..7dcee63 100644 --- a/cursorpaging-jpa/src/test/java/io/vigier/cursorpaging/jpa/itest/PostgreSqlCursorPageTest.java +++ b/cursorpaging-jpa/src/test/java/io/vigier/cursorpaging/jpa/itest/PostgreSqlCursorPageTest.java @@ -281,6 +281,30 @@ void shouldCountWhereFilterWithLikeExpressions() { assertThat( count ).isEqualTo( 2 ); } + @Test + void shouldAddCountInPageRequestWhenRequested() { + // Given + testDataGenerator.generateData( TestDataGenerator.NAMES.length ); + final PageRequest request = PageRequest.create( + b -> b.pageSize( 5 ).enableTotalCount( true ).asc( DataRecord_.id ) ); + + // When + final var page = dataRecordRepository.loadPage( request ); + + assertThat( page ).isNotNull(); + assertThat( page.getTotalCount() ).isPresent().get().isEqualTo( (long) TestDataGenerator.NAMES.length ); + + testDataGenerator.generateDataRecords( 66 ); + final var secondPage = dataRecordRepository.loadPage( page.next().orElseThrow() ); + + // THEN the totalCount should not be re-calculated + assertThat( secondPage.getTotalCount() ).isPresent().get().isEqualTo( (long) TestDataGenerator.NAMES.length ); + + final var thirdPage = dataRecordRepository.loadPage( + secondPage.next().orElseThrow().withEnableTotalCount( true ) ); + assertThat( thirdPage.getTotalCount() ).isPresent().get().isEqualTo( TestDataGenerator.NAMES.length + 66L ); + } + @Test void shouldReturnTotalCountWhenNoFilterPresent() { testDataGenerator.generateData( 42 ); @@ -290,7 +314,7 @@ void shouldReturnTotalCountWhenNoFilterPresent() { final var count = dataRecordRepository.count( request ); - assertThat( count ).isEqualTo( 42 ); + assertThat( count ).isEqualTo( 42L ); } @Test