Merge pull request #29 from alleyinteractive/cursor

src/class-runner.php

@@ -7,6 +7,7 @@
 
 namespace Feed_Consumer;
 
+use Feed_Consumer\Contracts\With_Cursor;
 use Monolog\Logger;
 use Psr\Log\LoggerInterface;
 use RuntimeException;
@@ -77,6 +78,15 @@ public static function processor( int $feed_id ): Contracts\Processor {
 
 		$processor->set_settings( $settings[ Settings::escape_setting_name( $settings['processor'] ) ] ?? [] );
 
+		// Instantiate the processor's cursor if supported.
+		if ( $processor instanceof With_Cursor ) {
+			$cursor = get_post_meta( $feed_id, With_Cursor::CURSOR_META_KEY, true );
+
+			if ( ! is_null( $cursor ) && '' !== $cursor ) {
+				$processor->set_cursor( (string) $cursor );
+			}
+		}
+
 		return $processor;
 	}
 
@@ -288,6 +298,17 @@ public function run() {
 		 */
 		do_action( 'feed_consumer_run_complete', $this->feed_id, $loaded_data, $processor::class );
 
+		// Store the cursor of the processor.
+		if ( $processor instanceof With_Cursor ) {
+			$cursor = $processor->get_cursor();
+
+			if ( is_null( $cursor ) ) {
+				delete_post_meta( $this->feed_id, With_Cursor::CURSOR_META_KEY );
+			} else {
+				update_post_meta( $this->feed_id, With_Cursor::CURSOR_META_KEY, $cursor );
+			}
+		}
+
 		// Update the last run time of the feed.
 		update_post_meta( $this->feed_id, static::LAST_RUN_META_KEY, current_time( 'timestamp' ) ); // phpcs:ignore WordPress.DateTime.CurrentTimeTimestamp.Requested
 

src/contracts/interface-with-cursor.php

@@ -0,0 +1,35 @@
+<?php
+/**
+ * With_Cursor interface file.
+ *
+ * @package feed-consumer
+ */
+
+namespace Feed_Consumer\Contracts;
+
+/**
+ * With Cursor Interface
+ */
+interface With_Cursor {
+	/**
+	 * Cursor meta key.
+	 *
+	 * @var string
+	 */
+	public const CURSOR_META_KEY = 'feed_consumer_cursor';
+
+	/**
+	 * Retrieve the cursor.
+	 *
+	 * @return string|null
+	 */
+	public function get_cursor(): ?string;
+
+	/**
+	 * Set the cursor.
+	 *
+	 * @param string $cursor Cursor to set.
+	 * @return static
+	 */
+	public function set_cursor( string $cursor ): static;
+}

src/processor/class-rss-processor.php

@@ -7,6 +7,7 @@
 
 namespace Feed_Consumer\Processor;
 
+use Feed_Consumer\Contracts\With_Cursor;
 use Feed_Consumer\Extractor\Feed_Extractor;
 use Feed_Consumer\Loader\Post_Loader;
 use Feed_Consumer\Transformer\RSS_Transformer;
@@ -16,7 +17,9 @@
  *
  * Extracts an array of items from an RSS feed.
  */
-class RSS_Processor extends Processor {
+class RSS_Processor extends Processor implements With_Cursor {
+	use Cursor;
+
 	/**
 	 * Constructor.
 	 */

src/processor/trait-cursor.php

@@ -0,0 +1,41 @@
+<?php
+/**
+ * Cursor class file
+ *
+ * @package feed-consumer
+ */
+
+namespace Feed_Consumer\Processor;
+
+/**
+ * Processor Cursor Storage
+ */
+trait Cursor {
+	/**
+	 * Cursor storage.
+	 *
+	 * @var string|null
+	 */
+	protected ?string $cursor = null;
+
+	/**
+	 * Retrieve the cursor.
+	 *
+	 * @return string|null
+	 */
+	public function get_cursor(): ?string {
+		return $this->cursor;
+	}
+
+	/**
+	 * Set the cursor.
+	 *
+	 * @param string $cursor Cursor to set.
+	 * @return static
+	 */
+	public function set_cursor( string $cursor ): static {
+		$this->cursor = $cursor;
+
+		return $this;
+	}
+}

src/transformer/class-rss-transformer.php

@@ -28,6 +28,7 @@ public function presets(): array {
 		return (array) apply_filters(
 			'feed_consumer_rss_transformer_presets',
 			[
+				static::PATH_CURSOR            => 'pubDate',
 				static::PATH_ITEMS             => '/rss/channel/item',
 				static::PATH_GUID              => 'guid',
 				static::PATH_TITLE             => 'title',

src/transformer/class-transformer.php

@@ -10,6 +10,7 @@
 use Feed_Consumer\Contracts\Extractor;
 use Feed_Consumer\Contracts\Processor;
 use Feed_Consumer\Contracts\Transformer as Contract;
+use Feed_Consumer\Contracts\With_Cursor;
 use Feed_Consumer\Contracts\With_Extractor;
 use Feed_Consumer\Contracts\With_Processor;
 
@@ -94,6 +95,13 @@ abstract class Transformer implements With_Extractor, With_Processor, Contract {
 	 */
 	public const PATH_IMAGE_CREDIT = 'path_image_credit';
 
+	/**
+	 * XPath key for the item cursor (date or ID).
+	 *
+	 * @var string
+	 */
+	public const PATH_CURSOR = 'path_cursor';
+
 	/**
 	 * Settings key to not convert to Gutenberg blocks.
 	 *
@@ -104,7 +112,7 @@ abstract class Transformer implements With_Extractor, With_Processor, Contract {
 	/**
 	 * Processor instance.
 	 *
-	 * @var Processor|null
+	 * @var Processor|With_Cursor|null
 	 */
 	protected ?Processor $processor;
 

src/transformer/class-xml-transformer.php

@@ -8,12 +8,15 @@
 namespace Feed_Consumer\Transformer;
 
 use Alley\WP\Block_Converter\Block_Converter;
+use Feed_Consumer\Contracts\With_Cursor;
 use Feed_Consumer\Contracts\With_Presets;
 use Feed_Consumer\Contracts\With_Setting_Fields;
 use Feed_Consumer\Loader\Post_Loader;
 use Fieldmanager_TextField;
 use SimpleXMLElement;
 
+use function Mantle\Support\Helpers\collect;
+
 /**
  * XML Transformer
  *
@@ -35,6 +38,7 @@ public function setting_fields(): array {
 
 		return [
 			static::PATH_ITEMS             => new Fieldmanager_TextField( __( 'XPath to items', 'feed-consumer' ) ),
+			static::PATH_CURSOR            => new Fieldmanager_TextField( __( 'XPath to cursor field (date)', 'feed-consumer' ) ),
 			static::PATH_GUID              => new Fieldmanager_TextField( __( 'XPath to guid', 'feed-consumer' ) ),
 			static::PATH_TITLE             => new Fieldmanager_TextField( __( 'XPath to title', 'feed-consumer' ) ),
 			static::PATH_PERMALINK         => new Fieldmanager_TextField( __( 'XPath to permalink', 'feed-consumer' ) ),
@@ -91,22 +95,69 @@ public function data(): array {
 			return [];
 		}
 
-		return array_map(
-			fn ( SimpleXMLElement $item ) => [
-				Post_Loader::BYLINE            => $this->extract_by_xpath( $item, $settings[ static::PATH_BYLINE ] ?? 'author' ),
-				Post_Loader::CONTENT           => empty( $settings[ static::DONT_CONVERT_TO_BLOCKS ] )
-					? (string) new Block_Converter( $this->extract_by_xpath( $item, $settings[ static::PATH_CONTENT ] ?? 'description' ) )
-					: $this->extract_by_xpath( $item, $settings[ static::PATH_CONTENT ] ?? 'description' ),
-				Post_Loader::GUID              => $this->extract_by_xpath( $item, $settings[ static::PATH_GUID ] ?? 'guid' ),
-				Post_Loader::IMAGE             => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE ] ?? 'image' ),
-				Post_Loader::IMAGE_CAPTION     => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_CAPTION ] ?? 'image_caption' ),
-				Post_Loader::IMAGE_CREDIT      => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_CREDIT ] ?? 'image_credit' ),
-				Post_Loader::IMAGE_DESCRIPTION => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_DESCRIPTION ] ?? 'image_description' ),
-				Post_Loader::PERMALINK         => $this->extract_by_xpath( $item, $settings[ static::PATH_PERMALINK ] ?? 'link' ),
-				Post_Loader::TITLE             => $this->extract_by_xpath( $item, $settings[ static::PATH_TITLE ] ?? 'title' ),
-			],
-			(array) $items,
-		);
+		$processor_cursor = null;
+
+		// Determine the processor's cursor timestamp.
+		if ( $this->processor && $this->processor instanceof With_Cursor ) {
+			$processor_cursor = $this->processor->get_cursor();
+
+			// Support a numeric cursor OR a date cursor.
+			if ( ! is_null( $processor_cursor ) && is_numeric( $processor_cursor ) ) {
+				$processor_cursor = (int) $processor_cursor;
+			} elseif ( ! is_null( $processor_cursor ) ) {
+				$processor_cursor = strtotime( $processor_cursor );
+			}
+		}
+
+		$items = collect( (array) $items )
+			->map(
+				fn ( SimpleXMLElement $item ) => [
+					'cursor'                       => $this->extract_by_xpath( $item, $settings[ static::PATH_CURSOR ] ?? '' ),
+					Post_Loader::BYLINE            => $this->extract_by_xpath( $item, $settings[ static::PATH_BYLINE ] ?? 'author' ),
+					Post_Loader::CONTENT           => empty( $settings[ static::DONT_CONVERT_TO_BLOCKS ] )
+						? (string) new Block_Converter( $this->extract_by_xpath( $item, $settings[ static::PATH_CONTENT ] ?? 'description' ) )
+						: $this->extract_by_xpath( $item, $settings[ static::PATH_CONTENT ] ?? 'description' ),
+					Post_Loader::GUID              => $this->extract_by_xpath( $item, $settings[ static::PATH_GUID ] ?? 'guid' ),
+					Post_Loader::IMAGE             => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE ] ?? 'image' ),
+					Post_Loader::IMAGE_CAPTION     => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_CAPTION ] ?? 'image_caption' ),
+					Post_Loader::IMAGE_CREDIT      => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_CREDIT ] ?? 'image_credit' ),
+					Post_Loader::IMAGE_DESCRIPTION => $this->extract_by_xpath( $item, $settings[ static::PATH_IMAGE_DESCRIPTION ] ?? 'image_description' ),
+					Post_Loader::PERMALINK         => $this->extract_by_xpath( $item, $settings[ static::PATH_PERMALINK ] ?? 'link' ),
+					Post_Loader::TITLE             => $this->extract_by_xpath( $item, $settings[ static::PATH_TITLE ] ?? 'title' ),
+				],
+			)
+			->filter(
+				function ( array $item ) use ( $processor_cursor ) {
+					// Check if the processor supports a cursor or if one is set.
+					if ( is_null( $processor_cursor ) ) {
+						return true;
+					}
+
+					// Check if the item has a cursor set.
+					if ( ! isset( $item['cursor'] ) ) {
+						return true;
+					}
+
+					// Check if the item's cursor is newer than the processor's cursor.
+					$cursor = is_numeric( $item['cursor'] )
+						? (int) $item['cursor']
+						: strtotime( $item['cursor'] );
+
+					return $cursor > $processor_cursor;
+				}
+			)
+			->values();
+
+		// Update the processor's cursor if supported.
+		if ( $this->processor && $this->processor instanceof With_Cursor ) {
+			$last_item = $items->last();
+
+			if ( ! empty( $last_item['cursor'] ) ) {
+				$this->processor->set_cursor( $last_item['cursor'] );
+			}
+		}
+
+		return $items->all();
 	}
 
 	/**

tests/class-test-case.php

@@ -4,7 +4,9 @@
 use Feed_Consumer\Contracts\Extractor;
 use Feed_Consumer\Contracts\Processor as Processor_Contract;
 use Feed_Consumer\Contracts\Transformer;
+use Feed_Consumer\Contracts\With_Cursor;
 use Feed_Consumer\Loader\Post_Loader;
+use Feed_Consumer\Processor\Cursor;
 use Feed_Consumer\Processor\Processor;
 use Mantle\Http_Client\Response;
 use Mantle\Testing\Concerns\With_Faker;
@@ -24,7 +26,9 @@ public function setUp(): void {
 	}
 
 	protected function make_processor( array $settings = [] ): Processor {
-		$instance = new class() extends Processor {
+		$instance = new class() extends Processor implements With_Cursor {
+			use Cursor;
+
 			public function name(): string {
 				return 'Test Processor';
 			}

tests/processor/test-cursor.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace Feed_Consumer\Tests\Processor;
+
+use Feed_Consumer\Loader\Post_Loader;
+use Feed_Consumer\Processor\RSS_Processor;
+use Feed_Consumer\Runner;
+use Feed_Consumer\Settings;
+use Feed_Consumer\Tests\Test_Case;
+use Mantle\Testing\Concerns\Refresh_Database;
+use Mantle\Testing\Mock_Http_Response;
+
+/**
+ * @group processor
+ */
+class Cursor_Test extends Test_Case {
+	public function get_default_cursor() {
+		$processor = $this->make_processor();
+
+		$this->assertNull( $processor->get_cursor() );
+	}
+
+	public function test_get_cursor() {
+		$processor = $this->make_processor();
+
+		$processor->set_cursor( '123' );
+
+		$this->assertEquals( '123', $processor->get_cursor() );
+	}
+
+	public function test_set_cursor() {
+		$processor = $this->make_processor();
+
+		$this->assertNull( $processor->get_cursor() );
+
+		$processor->set_cursor( '123' );
+
+		$this->assertEquals( '123', $processor->get_cursor() );
+	}
+}

tests/processor/test-rss-processor.php

@@ -57,6 +57,8 @@ public function test_load_rss_feed() {
 				]
 			);
 
+		$this->assertNull( Runner::processor( $feed_id )->get_cursor() );
+
 		Runner::run_scheduled( $feed_id );
 
 		$this->assertPostExists(
@@ -69,6 +71,8 @@ public function test_load_rss_feed() {
 		$this->assertNotEmpty( get_post_meta( $feed_id, Runner::LAST_RUN_META_KEY, true ) );
 
 		$this->assertInCronQueue( Runner::CRON_HOOK, [ $feed_id ] );
+
+		$this->assertNotNull( Runner::processor( $feed_id )->get_cursor() );
 	}
 
 	public function test_handle_rss_feed_error() {

tests/transformer/test-rss-transformer.php

@@ -64,4 +64,27 @@ public function test_rss_transformation_error() {
 
 		$this->assertCount( 0, $data );
 	}
+
+	public function test_rss_transformer_with_cursor() {
+		$processor = $this->make_processor();
+
+		// Setting the cursor relative to the 3rd item in the feed (it should only include 1-2).
+		$processor->set_cursor( 'Fri, 08 Apr 2022 19:18:04 +0000' );
+
+		$extractor = $this->make_extractor(
+			Mock_Http_Response::create()
+				->with_header( 'Content-Type', 'application/rss+xml' )
+				->with_body( file_get_contents( __DIR__ . '/../fixtures/rss-feed.xml' ) ),
+			$processor,
+		);
+
+		$transformer = new RSS_Transformer( $processor, $extractor );
+
+		$transformer->set_processor( $processor );
+		$transformer->set_extractor( $extractor );
+
+		$data = $transformer->data();
+
+		$this->assertCount( 2, $data );
+	}
 }