Skip to content

Commit

Permalink
!!![TASK] Clean and optimize frontend helper: PageIndexer
Browse files Browse the repository at this point in the history
Cleans and optimizes the frontend helper "PageIndexer" integration
test.

Failure test for non existing pages/sites is refactored and
PageIndexer adapted, even if AfterCacheableContentIsGeneratedEvent
is not dispatched e.g. as the page is not found, the response will
always contain the pageIndexed flag.

Additionally the methods of PageIndexer that were defined as public
to simplify testing are now protected, the tests are adapted
accordingly. If you have used this methods, you have to adapt your
code. Affected methods:
- setupConfiguration
- index
- indexPage

Relates: #3735
  • Loading branch information
dkd-friedrich committed Oct 6, 2023
1 parent e602789 commit e6a6981
Show file tree
Hide file tree
Showing 18 changed files with 271 additions and 853 deletions.
28 changes: 21 additions & 7 deletions Classes/IndexQueue/FrontendHelper/PageIndexer.php
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,9 @@ public function __invoke(AfterCacheableContentIsGeneratedEvent $event): void
}

/**
* @internal currently only public for tests
* Index item
*/
public function index(Item $indexQueueItem, TypoScriptFrontendController $tsfe): void
protected function index(Item $indexQueueItem, TypoScriptFrontendController $tsfe): void
{
$this->solrConnection = $this->getSolrConnection($indexQueueItem, $tsfe->getLanguage(), $this->configuration->getLoggingExceptions());

Expand Down Expand Up @@ -291,7 +291,7 @@ protected function getPageDocument(TypoScriptFrontendController $tsfe, string $u
*
* @return bool TRUE after successfully indexing the page, FALSE on error
*/
public function indexPage(
protected function indexPage(
Document $pageDocument,
Item $indexQueueItem,
TypoScriptFrontendController $tsfe,
Expand Down Expand Up @@ -366,11 +366,13 @@ protected function addDocumentsToSolrIndex(array $documents): bool
}

/**
* @internal only used for tests
* Initialize PageIndexer
*
* As the Solr configuration initialization might affect the request
* we cannot initialize the configuration directly on activation
*/
public function setupConfiguration(): void
protected function setupConfiguration(): void
{
// currently needed for tests, will be separated.
$this->logger = new SolrLogManager(__CLASS__, GeneralUtility::makeInstance(DebugWriter::class));
$this->configuration = Util::getSolrConfiguration();
}
Expand All @@ -385,7 +387,19 @@ protected function getEventDispatcher(): EventDispatcherInterface
*/
public function deactivate(PageIndexerResponse $response): void
{
if ($this->activated) {
if (!isset($this->responseData['pageIndexed'])) {
$this->responseData['pageIndexed'] = false;
}
$response->addActionResult($this->action, $this->responseData);

$this->setupConfiguration();
if ($this->configuration->getLoggingExceptions()) {
$this->logger->error(
'Unknown exception while trying to index page',
);
}
}
$this->activated = false;
$response->addActionResult($this->action, $this->responseData);
}
}
10 changes: 9 additions & 1 deletion Documentation/Releases/solr-release-12-0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,18 @@ If you've used this class or the SolrConnection directly, you have to adapt your
- use \Solarium\Core\Client\Endpoint instead of \ApacheSolrForTypo3\Solr\System\Solr\Node
- call \ApacheSolrForTypo3\Solr\System\Solr\SolrConnection->getEndpoint() instead of \ApacheSolrForTypo3\Solr\System\Solr\SolrConnection\getNode(),
method will return Solarium Endpoint
- Node could be converted to string to get the core base URI, getCoreBaseUri() can be used instead.
- Node could be converted to string to get the core base URI, getCoreBaseUri() can be used instead.

Note: With dropping the Node implementation we also dropped the backwards compatibility that allows to define the Solr path segment "/solr" within "solr_path_read" or "solr_path_write". Be sure your configuration doesn't contain this path segment!

!!! Changed visibility of ApacheSolrForTypo3\Solr\IndexQueue\FrontendHelper\PageIndexer methods
-----------------------------------------------------------------------------------------------

For testing purposes some methods of the PageIndexer were defined as public, these methods are now protected. The tests are adapted accordingly, so that there is no need to declare the methods as public.
If you have used one of this methods, you have to adapt your code. Affected methods:
- setupConfiguration
- index
- indexPage

Frontend Helper Changes
-----------------------
Expand Down
56 changes: 6 additions & 50 deletions Tests/Integration/Fixtures/Search/can_search.csv
Original file line number Diff line number Diff line change
@@ -1,50 +1,6 @@
"pages",,,,,,
,"uid","is_siteroot","doktype","title","slug","hidden"
,1,1,1,"Hello Search Test","/",0
"sys_template",,,,,,
,"uid","pid","root","clear","sorting","config",
,1,1,1,3,100,"
page = PAGE
page.typeNum = 0

plugin.tx_solr {

enabled = 1

index {
fieldProcessingInstructions {
changed = timestampToIsoDate
created = timestampToIsoDate
endtime = timestampToUtcIsoDate
rootline = pageUidToHierarchy
}

queue {

// mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing)

pages = 1
pages {
initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page

// allowed page types (doktype) when indexing pages
allowedPageTypes = 1,7

indexingPriority = 0

indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer
indexer {
// add options for the indexer here
}

// Only index standard pages and mount points that are not overlayed.
additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0

fields {
sortSubTitle_stringS = subtitle
}
}

}
}
}"
"pages",
,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp"
,2,1,1,1,"/hello_solr","Hello Search Test","the subtitle",1449151778,1449151778
"tx_solr_indexqueue_item",
,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors"
,2,1,"pages",2,"pages",1449151778,0,0,0,0,0
94 changes: 32 additions & 62 deletions Tests/Integration/Fixtures/Search/phrase_search.csv
Original file line number Diff line number Diff line change
@@ -1,64 +1,34 @@
"pages",,,,,,,
,"uid","pid","is_siteroot","doktype","hidden","slug","title"
,1,0,1,1,0,"/","World Hello Search Test"
,2,1,0,1,0,"/improve","Hello you can improve the precision by using phrase configuration World"
,3,1,0,1,0,"/hello-phrase-search-world","Hello phrase search World"
,4,1,0,1,0,"/hello-test-wonderful-world","Hello Test wonderful World"
,5,1,0,1,0,"/hello-wonderful-test-world","Hello wonderful Test World"
,6,1,0,1,0,"/bigram-phrases","Bigraming phrases strip down the sentence to the two-word combinations."
,7,1,0,1,0,"/trigram-phrases","Trigraming phrases strip down the sentence to triplets phrases"
,8,1,0,1,0,"/difference-to-bi-is-tri","Difference to bi is tri, which means building of triplets phrases."
,9,1,0,1,0,"/hello-solr-wolrd","Hello Solr Wolrd"
,10,1,0,1,0,"/hello-test-wolrd","Hello Test Wolrd"
,11,1,0,1,0,"/test-hello-solr-world","Test Hello Solr World"
,12,1,0,1,0,"/test-hello-search-world","Test Hello Search World"
,13,1,0,1,0,"/hello-the-test-search-world","Hello the Test Search World"
,14,1,0,1,0,"/hello-world-for-phrase-searching","Hello World for phrase searching"
,15,1,0,1,0,"/about","Solr is blazing-fast and open source enterprise search platform built on Apache Lucene"
"sys_template",,,,,,,
,"uid","pid","root","clear","sorting","constants","config",
,1,1,1,3,100,"","
page = PAGE
page.typeNum = 0

plugin.tx_solr {
search.query.phrase = 1
enabled = 1

index {
fieldProcessingInstructions {
changed = timestampToIsoDate
created = timestampToIsoDate
endtime = timestampToUtcIsoDate
rootline = pageUidToHierarchy
}

queue {

// mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing)

pages = 1
pages {
initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page

// allowed page types (doktype) when indexing pages
allowedPageTypes = 1,7

indexingPriority = 0

indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer
indexer {
// add options for the indexer here
}

// Only index standard pages and mount points that are not overlayed.
additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0

fields {
sortSubTitle_stringS = subtitle
}
}

}
}
}",
,2,1,1,1,0,"/","World Hello Search Test"
,3,1,0,1,0,"/improve","Hello you can improve the precision by using phrase configuration World"
,4,1,0,1,0,"/hello-phrase-search-world","Hello phrase search World"
,5,1,0,1,0,"/hello-test-wonderful-world","Hello Test wonderful World"
,6,1,0,1,0,"/hello-wonderful-test-world","Hello wonderful Test World"
,7,1,0,1,0,"/bigram-phrases","Bigraming phrases strip down the sentence to the two-word combinations."
,8,1,0,1,0,"/trigram-phrases","Trigraming phrases strip down the sentence to triplets phrases"
,9,1,0,1,0,"/difference-to-bi-is-tri","Difference to bi is tri, which means building of triplets phrases."
,10,1,0,1,0,"/hello-solr-wolrd","Hello Solr Wolrd"
,11,1,0,1,0,"/hello-test-wolrd","Hello Test Wolrd"
,12,1,0,1,0,"/test-hello-solr-world","Test Hello Solr World"
,13,1,0,1,0,"/test-hello-search-world","Test Hello Search World"
,14,1,0,1,0,"/hello-the-test-search-world","Hello the Test Search World"
,15,1,0,1,0,"/hello-world-for-phrase-searching","Hello World for phrase searching"
,16,1,0,1,0,"/about","Solr is blazing-fast and open source enterprise search platform built on Apache Lucene"
"tx_solr_indexqueue_item",
,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors"
,2,1,"pages",2,"pages",1449151778,0,0,0,0,0
,3,1,"pages",3,"pages",1449151778,0,0,0,0,0
,4,1,"pages",4,"pages",1449151778,0,0,0,0,0
,5,1,"pages",5,"pages",1449151778,0,0,0,0,0
,6,1,"pages",6,"pages",1449151778,0,0,0,0,0
,7,1,"pages",7,"pages",1449151778,0,0,0,0,0
,8,1,"pages",8,"pages",1449151778,0,0,0,0,0
,9,1,"pages",9,"pages",1449151778,0,0,0,0,0
,10,1,"pages",10,"pages",1449151778,0,0,0,0,0
,11,1,"pages",11,"pages",1449151778,0,0,0,0,0
,12,1,"pages",12,"pages",1449151778,0,0,0,0,0
,13,1,"pages",13,"pages",1449151778,0,0,0,0,0
,14,1,"pages",14,"pages",1449151778,0,0,0,0,0
,15,1,"pages",15,"pages",1449151778,0,0,0,0,0
,16,1,"pages",16,"pages",1449151778,0,0,0,0,0
94 changes: 32 additions & 62 deletions Tests/Integration/Fixtures/Search/phrase_search_bigram.csv
Original file line number Diff line number Diff line change
@@ -1,68 +1,38 @@
"pages",,,,,,,
,"uid","pid","is_siteroot","doktype","hidden","slug","title"
# 100% matching
,1,0,1,1,0,"/","Bigram Phrase Search"
,2,1,1,1,0,"/","Bigram Phrase Search"
# bigram phrase slop = 0
,2,1,0,1,0,"/dump-sentence","This is some dump sentence to match Bigram Phrase"
,3,1,0,1,0,"/match-phrase-search","This is some dump sentence to match Phrase Search"
,3,1,0,1,0,"/dump-sentence","This is some dump sentence to match Bigram Phrase"
,4,1,0,1,0,"/match-phrase-search","This is some dump sentence to match Phrase Search"
# bigram phrase slop = 1
,4,1,0,1,0,"/slop-1-bigram-gag-phrase","This is some dump sentence to match slop 1 Bigram gag Phrase"
,5,1,0,1,0,"/slop-1-phrase-gag-search","This is some dump sentence to match slop 1 Phrase gag Search"
,5,1,0,1,0,"/slop-1-bigram-gag-phrase","This is some dump sentence to match slop 1 Bigram gag Phrase"
,6,1,0,1,0,"/slop-1-phrase-gag-search","This is some dump sentence to match slop 1 Phrase gag Search"
# bigram phrase slop = 2
,6,1,0,1,0,"/slop-2-bigram-gag-gag-phrase","This is some dump sentence to match slop 2 Bigram gag gag Phrase"
,7,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Phrase gag gag Search"
,8,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs"
,9,1,0,1,0,"/no-match","This is some dump sentence to match nothing"
,10,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0."
,11,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only"
,12,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only"
,13,1,0,1,0,"/match-bigram-only","This is some dump sentence to match Bigram only"
,14,1,0,1,0,"/implicit-bigram-phrase-pf2","Implicit Bigram gag gag gag gag Phrase gag gag gag gag searches in pf2 fields."
,15,1,0,1,0,"/bigram-gag-gag-gag-gag-phrase","Bigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature."
"sys_template",,,,,,,
,"uid","pid","root","clear","sorting","constants","config",
,1,1,1,3,100,"","
page = PAGE
page.typeNum = 0

plugin.tx_solr {
search.query.bigramPhrase = 1
enabled = 1

index {
fieldProcessingInstructions {
changed = timestampToIsoDate
created = timestampToIsoDate
endtime = timestampToUtcIsoDate
rootline = pageUidToHierarchy
}

queue {

// mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing)

pages = 1
pages {
initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page

// allowed page types (doktype) when indexing pages
allowedPageTypes = 1,7

indexingPriority = 0

indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer
indexer {
// add options for the indexer here
}

// Only index standard pages and mount points that are not overlayed.
additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0

fields {
sortSubTitle_stringS = subtitle
}
}

}
}
}"
,7,1,0,1,0,"/slop-2-bigram-gag-gag-phrase","This is some dump sentence to match slop 2 Bigram gag gag Phrase"
,8,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Phrase gag gag Search"
,9,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs"
,10,1,0,1,0,"/no-match","This is some dump sentence to match nothing"
,11,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0."
,12,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only"
,13,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only"
,14,1,0,1,0,"/match-bigram-only","This is some dump sentence to match Bigram only"
,15,1,0,1,0,"/implicit-bigram-phrase-pf2","Implicit Bigram gag gag gag gag Phrase gag gag gag gag searches in pf2 fields."
,16,1,0,1,0,"/bigram-gag-gag-gag-gag-phrase","Bigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature."
"tx_solr_indexqueue_item",
,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors"
,2,1,"pages",2,"pages",1449151778,0,0,0,0,0
,3,1,"pages",3,"pages",1449151778,0,0,0,0,0
,4,1,"pages",4,"pages",1449151778,0,0,0,0,0
,5,1,"pages",5,"pages",1449151778,0,0,0,0,0
,6,1,"pages",6,"pages",1449151778,0,0,0,0,0
,7,1,"pages",7,"pages",1449151778,0,0,0,0,0
,8,1,"pages",8,"pages",1449151778,0,0,0,0,0
,9,1,"pages",9,"pages",1449151778,0,0,0,0,0
,10,1,"pages",10,"pages",1449151778,0,0,0,0,0
,11,1,"pages",11,"pages",1449151778,0,0,0,0,0
,12,1,"pages",12,"pages",1449151778,0,0,0,0,0
,13,1,"pages",13,"pages",1449151778,0,0,0,0,0
,14,1,"pages",14,"pages",1449151778,0,0,0,0,0
,15,1,"pages",15,"pages",1449151778,0,0,0,0,0
,16,1,"pages",16,"pages",1449151778,0,0,0,0,0
Loading

0 comments on commit e6a6981

Please sign in to comment.