Use Solr OCR Highlighting Plugin in Search in Document Plugin

Classes/Common/Document.php

@@ -570,8 +570,8 @@ public static function &getInstance($uid, $pid = 0, $forceReload = false)
             if (!empty($extConf['caching'])) {
                 Helper::saveToSession(self::$registry, get_class($instance));
             }
+            $instance->logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(get_class($instance));
         }
-        $instance->logger = GeneralUtility::makeInstance(LogManager::class)->getLogger(get_class($instance));
         // Return new instance.
         return $instance;
     }
@@ -638,10 +638,8 @@ public function getPhysicalPage($logicalPage)
     }
 
     /**
-     * This extracts the raw text for a physical structure node / IIIF Manifest / Canvas. Text might be
-     * given as ALTO for METS or as annotations or ALTO for IIIF resources. If IIIF plain text annotations
-     * with the motivation "painting" should be treated as full text representations, the extension has to be
-     * configured accordingly.
+     * This extracts the OCR full text for a physical structure node / IIIF Manifest / Canvas. Text might be
+     * given as ALTO for METS or as annotations or ALTO for IIIF resources.
      *
      * @access public
      *
@@ -650,23 +648,23 @@ public function getPhysicalPage($logicalPage)
      * @param string $id: The @ID attribute of the physical structure node (METS) or the @id property
      * of the Manifest / Range (IIIF)
      *
-     * @return string The physical structure node's / IIIF resource's raw text
+     * @return string The OCR full text
      */
-    public abstract function getRawText($id);
+    public abstract function getFullText($id);
 
     /**
-     * This extracts the raw text for a physical structure node / IIIF Manifest / Canvas from an
-     * XML fulltext representation (currently only ALTO). For IIIF manifests, ALTO documents have
+     * This extracts the OCR full text for a physical structure node / IIIF Manifest / Canvas from an
+     * XML full text representation (currently only ALTO). For IIIF manifests, ALTO documents have
      * to be given in the Canvas' / Manifest's "seeAlso" property.
      *
      * @param string $id: The @ID attribute of the physical structure node (METS) or the @id property
      * of the Manifest / Range (IIIF)
      *
-     * @return string The physical structure node's / IIIF resource's raw text from XML
+     * @return string The OCR full text
      */
-    protected function getRawTextFromXml($id)
+    protected function getFullTextFromXml($id)
     {
-        $rawText = '';
+        $fullText = '';
         // Load available text formats, ...
         $this->loadFormats();
         // ... physical structure ...
@@ -677,54 +675,54 @@ protected function getRawTextFromXml($id)
         if (!empty($this->physicalStructureInfo[$id])) {
             while ($fileGrpFulltext = array_shift($fileGrpsFulltext)) {
                 if (!empty($this->physicalStructureInfo[$id]['files'][$fileGrpFulltext])) {
-                    // Get fulltext file.
-                    $file = GeneralUtility::getUrl($this->getFileLocation($this->physicalStructureInfo[$id]['files'][$fileGrpFulltext]));
-                    if ($file !== false) {
-                        // Turn off libxml's error logging.
-                        $libxmlErrors = libxml_use_internal_errors(true);
-                        // Disables the functionality to allow external entities to be loaded when parsing the XML, must be kept.
-                        $previousValueOfEntityLoader = libxml_disable_entity_loader(true);
-                        // Load XML from file.
-                        $rawTextXml = simplexml_load_string($file);
-                        // Reset entity loader setting.
-                        libxml_disable_entity_loader($previousValueOfEntityLoader);
-                        // Reset libxml's error logging.
-                        libxml_use_internal_errors($libxmlErrors);
-                        // Get the root element's name as text format.
-                        $textFormat = strtoupper($rawTextXml->getName());
+                    // Get full text file.
+                    $fileContent = GeneralUtility::getUrl($this->getFileLocation($this->physicalStructureInfo[$id]['files'][$fileGrpFulltext]));
+                    if ($fileContent !== false) {
+                        $textFormat = $this->getTextFormat($fileContent);
                     } else {
-                        $this->logger->warning('Couldn\'t load fulltext file for structure node @ID "' . $id . '"');
-                        return $rawText;
+                        $this->logger->warning('Couldn\'t load full text file for structure node @ID "' . $id . '"');
+                        return $fullText;
                     }
                     break;
                 }
             }
         } else {
             $this->logger->warning('Invalid structure node @ID "' . $id . '"');
-            return $rawText;
+            return $fullText;
         }
         // Is this text format supported?
-        if (
-            !empty($rawTextXml)
-            && !empty($this->formats[$textFormat])
-        ) {
-            if (!empty($this->formats[$textFormat]['class'])) {
-                $class = $this->formats[$textFormat]['class'];
-                // Get the raw text from class.
-                if (
-                    class_exists($class)
-                    && ($obj = GeneralUtility::makeInstance($class)) instanceof FulltextInterface
-                ) {
-                    $rawText = $obj->getRawText($rawTextXml);
-                    $this->rawTextArray[$id] = $rawText;
-                } else {
-                    $this->logger->warning('Invalid class/method "' . $class . '->getRawText()" for text format "' . $textFormat . '"');
-                }
-            }
+        // This part actually differs from previous version of indexed OCR
+        if (!empty($fileContent) && !empty($this->formats[$textFormat])) {
+            $fullText = $fileContent;
         } else {
             $this->logger->warning('Unsupported text format "' . $textFormat . '" in physical node with @ID "' . $id . '"');
         }
-        return $rawText;
+        return $fullText;
+    }
+
+    /**
+     * Get format of the OCR full text
+     *
+     * @access private
+     *
+     * @param string $fileContent: content of the XML file
+     *
+     * @return string The format of the OCR full text
+     */
+    private function getTextFormat($fileContent)
+    {
+        // Turn off libxml's error logging.
+        $libxmlErrors = libxml_use_internal_errors(true);
+        // Disables the functionality to allow external entities to be loaded when parsing the XML, must be kept.
+        $previousValueOfEntityLoader = libxml_disable_entity_loader(true);
+        // Load XML from file.
+        $rawTextXml = simplexml_load_string($fileContent);
+        // Reset entity loader setting.
+        libxml_disable_entity_loader($previousValueOfEntityLoader);
+        // Reset libxml's error logging.
+        libxml_use_internal_errors($libxmlErrors);
+        // Get the root element's name as text format.
+        return strtoupper($rawTextXml->getName());
     }
 
     /**
@@ -1306,9 +1304,14 @@ public function save($pid = 0, $core = 0, $owner = null)
         }
         // Add document to index.
         if ($core) {
-            Indexer::add($this, $core);
+            //TODO: change return of this method to true on success and false on failure
+            $hasErrors = Indexer::add($this, $core);
+            if ($hasErrors) {
+                return false;
+            }
         } else {
             $this->logger->notice('Invalid UID "' . $core . '" for Solr core');
+            return false;
         }
         return true;
     }

Classes/Common/DocumentList.php

@@ -12,8 +12,10 @@
 
 namespace Kitodo\Dlf\Common;
 
+use Kitodo\Dlf\Common\SolrSearchResult\ResultDocument;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerAwareTrait;
+use Solarium\QueryType\Select\Result\Result;
 use TYPO3\CMS\Core\SingletonInterface;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -237,74 +239,8 @@ protected function getRecord($element)
                 && $this->metadata['options']['source'] == 'search'
             ) {
                 if ($this->solrConnect()) {
-                    $fields = Solr::getFields();
-                    $params = [];
-                    // Restrict the fields to the required ones
-                    $params['fields'] = $fields['uid'] . ',' . $fields['id'] . ',' . $fields['toplevel'] . ',' . $fields['thumbnail'] . ',' . $fields['page'];
-                    foreach ($this->solrConfig as $solr_name) {
-                        $params['fields'] .= ',' . $solr_name;
-                    }
-                    // If it is a fulltext search, enable highlighting.
-                    if ($this->metadata['fulltextSearch']) {
-                        $params['component'] = [
-                            'highlighting' => [
-                                'query' => Solr::escapeQuery($this->metadata['searchString']),
-                                'field' => $fields['fulltext'],
-                                'usefastvectorhighlighter' => true
-                            ]
-                        ];
-                    }
-                    // Set additional query parameters.
-                    $params['start'] = 0;
-                    // Set reasonable limit for safety reasons.
-                    // We don't expect to get more than 10.000 hits per UID.
-                    $params['rows'] = 10000;
-                    // Take over existing filter queries.
-                    $params['filterquery'] = isset($this->metadata['options']['params']['filterquery']) ? $this->metadata['options']['params']['filterquery'] : [];
-                    // Extend filter query to get all documents with the same UID.
-                    foreach ($params['filterquery'] as $key => $value) {
-                        if (isset($value['query'])) {
-                            $params['filterquery'][$key]['query'] = $value['query'] . ' OR ' . $fields['toplevel'] . ':true';
-                        }
-                    }
-                    // Add filter query to get all documents with the required uid.
-                    $params['filterquery'][] = ['query' => $fields['uid'] . ':' . Solr::escapeQuery($record['uid'])];
-                    // Add sorting.
-                    $params['sort'] = $this->metadata['options']['params']['sort'];
-                    // Set query.
-                    $params['query'] = $this->metadata['options']['select'] . ' OR ' . $fields['toplevel'] . ':true';
-                    // Perform search for all documents with the same uid that either fit to the search or marked as toplevel.
-                    $selectQuery = $this->solr->service->createSelect($params);
-                    $result = $this->solr->service->select($selectQuery);
-                    // If it is a fulltext search, fetch the highlighting results.
-                    if ($this->metadata['fulltextSearch']) {
-                        $highlighting = $result->getHighlighting();
-                    }
-                    // Process results.
-                    foreach ($result as $resArray) {
-                        // Prepare document's metadata.
-                        $metadata = [];
-                        foreach ($this->solrConfig as $index_name => $solr_name) {
-                            if (!empty($resArray->$solr_name)) {
-                                $metadata[$index_name] = (is_array($resArray->$solr_name) ? $resArray->$solr_name : [$resArray->$solr_name]);
-                            }
-                        }
-                        // Add metadata to list elements.
-                        if ($resArray->toplevel) {
-                            $record['thumbnail'] = $resArray->thumbnail;
-                            $record['metadata'] = $metadata;
-                        } else {
-                            $highlightedDoc = !empty($highlighting) ? $highlighting->getResult($resArray->id) : null;
-                            $highlight = !empty($highlightedDoc) ? $highlightedDoc->getField($fields['fulltext'])[0] : '';
-                            $record['subparts'][$resArray->id] = [
-                                'uid' => $resArray->uid,
-                                'page' => $resArray->page,
-                                'preview' => $highlight,
-                                'thumbnail' => $resArray->thumbnail,
-                                'metadata' => $metadata
-                            ];
-                        }
-                    }
+                    $result = $this->getSolrResult($record);
+                    $record = $this->getSolrRecord($record, $result);
                 }
             }
             // Save record for later usage.
@@ -316,6 +252,122 @@ protected function getRecord($element)
         return $record;
     }
 
+    /**
+     * It gets SOLR result
+     *
+     * @access private
+     *
+     * @param array $record: for searched document
+     *
+     * @return Result
+     */
+    private function getSolrResult($record) {
+        $fields = Solr::getFields();
+
+        $query = $this->solr->service->createSelect();
+        // Restrict the fields to the required ones
+        $query->setFields($fields['uid'] .',' . $fields['id'] .',' . $fields['toplevel'] .',' . $fields['thumbnail'] .',' . $fields['page']);
+        foreach ($this->solrConfig as $solr_name) {
+            $query->addField($solr_name);
+        }
+        // Set additional query parameters.
+        // Set reasonable limit for safety reasons.
+        // We don't expect to get more than 10.000 hits per UID.
+        $query->setStart(0)->setRows(1000);
+        // Take over existing filter queries.
+        $filterQueries = isset($this->metadata['options']['params']['filterquery']) ? $this->metadata['options']['params']['filterquery'] : [];
+        // Extend filter query to get all documents with the same UID.
+        foreach ($filterQueries as $key => $value) {
+            if (isset($value['query'])) {
+                $filterQuery[$key] = $value['query'] . ' OR ' . $fields['toplevel'] . ':true';
+                $query->addFilterQuery($filterQuery);
+            }
+        }
+        // Add filter query to get all documents with the required uid.
+        $filterQuery['key'] = $fields['uid'] . ':' . Solr::escapeQuery($record['uid']);
+        $query->addFilterQuery($filterQuery);
+        // Add sorting.
+        $query->addSort('score', $this->metadata['options']['params']['sort']['score']);
+        // Set query.
+        $query->setQuery($this->metadata['options']['select'] . ' OR ' . $fields['toplevel'] . ':true');
+
+        // If it is a fulltext search, enable highlighting.
+        if ($this->metadata['fulltextSearch']) {
+            $query->setQuery(
+                $fields['fulltext'] . ':' . Solr::escapeQuery($this->metadata['searchString']));
+                // TODO:toplevel documents throw error in SOLR PLugin:
+                // java.lang.RuntimeException: Could not determine OCR format for sample ''
+                //. ' OR ' . $fields['toplevel'] . ':true');
+            $query->getHighlighting();
+        };
+
+        $solrRequest = $this->solr->service->createRequest($query);
+
+        // If it is a fulltext search, enable highlighting.
+        if ($this->metadata['fulltextSearch']) {
+            // field for which highlighting is going to be performed,
+            // is required if you want to have OCR highlighting
+            $solrRequest->addParam('hl.ocr.fl', $fields['fulltext']);
+            // return the coordinates of highlighted search as absolute coordinates
+            $solrRequest->addParam('hl.ocr.absoluteHighlights', 'on');
+            // max amount of snippets for a single page
+            $solrRequest->addParam('hl.snippets', 20);
+        }
+        // Perform search for all documents with the same uid that either fit to the search or marked as toplevel.
+        $response = $this->solr->service->executeRequest($solrRequest);
+        return $this->solr->service->createResult($query, $response);
+    }
+
+    /**
+     * It processes SOLR result into record, which is
+     * going to be displayed in the frontend list.
+     *
+     * @access private
+     *
+     * @param array $record: for searched document
+     * @param Result $result: found in the SOLR index
+     *
+     * @return array
+     */
+    private function getSolrRecord($record, $result) {
+        // If it is a fulltext search, fetch the highlighting results.
+        if ($this->metadata['fulltextSearch']) {
+            $data = $result->getData();
+            $highlighting = $data['ocrHighlighting'];
+        }
+
+        // Process results.
+        foreach ($result as $resArray) {
+            // Prepare document's metadata.
+            $metadata = [];
+            foreach ($this->solrConfig as $index_name => $solr_name) {
+                if (!empty($resArray->$solr_name)) {
+                    $metadata[$index_name] = (is_array($resArray->$solr_name) ? $resArray->$solr_name : [$resArray->$solr_name]);
+                }
+            }
+            // Add metadata to list elements.
+            if ($resArray->toplevel) {
+                $record['thumbnail'] = $resArray->thumbnail;
+                $record['metadata'] = $metadata;
+            } else {
+                $highlight = '';
+                if (!empty($highlighting)) {
+                    $resultDocument = new ResultDocument($resArray, $highlighting, Solr::getFields());
+                    $highlight = $resultDocument->getSnippets();
+                }
+
+                $record['subparts'][$resArray->id] = [
+                    'uid' => $resArray->uid,
+                    'page' => $resArray->page,
+                    'preview' => $highlight,
+                    'thumbnail' => $resArray->thumbnail,
+                    'metadata' => $metadata
+                ];
+            }
+        }
+        return $record;
+    }
+
     /**
      * This returns the current position
      * @see \Iterator::key()

Classes/Common/FulltextInterface.php

@@ -21,6 +21,7 @@
  * @access public
  * @abstract
  */
+//TODO: check if this is still needed when actually full text xml is indexed
 interface FulltextInterface
 {
     /**