Merge pull request #22 from webcoast-dk/feature/support-typo3-cms-12

[BUGFIX] TYPO3 CMS 12 compatibility
webcoast-dk · Jun 27, 2024 · 58d0a1f · 58d0a1f
2 parents 1a6d9f6 + 73d2d2c
commit 58d0a1f
Show file tree

Hide file tree

Showing 23 changed files with 208 additions and 286 deletions.
diff --git a/Classes/Controller/CrawlerController.php b/Classes/Controller/CrawlerController.php
@@ -53,7 +53,7 @@ public function processQueue($numberOfItemsPerRun)
                 self::CONFIGURATION_TABLE,
                 ['uid' => $itemArray['configuration']]
             );
-            $configuration = $configurationResult->fetch();
+            $configuration = $configurationResult->fetchAssociative();
             $className = TypeUtility::getClassForType($configuration['type']);
             $crawler = GeneralUtility::makeInstance($className);
             if (!$crawler instanceof CrawlerInterface) {

diff --git a/Classes/Controller/QueueController.php b/Classes/Controller/QueueController.php
@@ -41,7 +41,7 @@ public function fillQueue($configurations)
         $result = true;
         foreach ($configurations as $configurationUid) {
             $configurationResult = $connection->select(['*'], self::CONFIGURATION_TABLE, ['uid' => $configurationUid]);
-            $configuration = $configurationResult->fetch();
+            $configuration = $configurationResult->fetchAssociative();
             $className = TypeUtility::getClassForType($configuration['type']);
             $crawler = GeneralUtility::makeInstance($className);
             if (!$crawler instanceof QueueInterface) {
@@ -60,4 +60,4 @@ public function fillQueue($configurations)
 
         return $result;
     }
-}
+}
diff --git a/Classes/Crawler/Files.php b/Classes/Crawler/Files.php
@@ -4,14 +4,13 @@
 
 use Doctrine\DBAL\DBALException;
 use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
 use TYPO3\CMS\Core\Database\Query\Restriction\FrontendRestrictionContainer;
-use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction;
 use TYPO3\CMS\Core\Resource\File;
 use TYPO3\CMS\Core\Resource\FileRepository;
 use TYPO3\CMS\Core\Resource\Folder;
 use TYPO3\CMS\Core\Resource\ResourceStorage;
 use TYPO3\CMS\Core\Resource\StorageRepository;
+use TYPO3\CMS\Core\Site\SiteFinder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\IndexedSearch\Indexer;
 use WEBcoast\VersatileCrawler\Controller\CrawlerController;
@@ -35,14 +34,7 @@ public function __construct()
         $this->storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
     }
 
-    /**
-     * @param array $configuration
-     * @param array $rootConfiguration
-     *
-     * @return boolean
-     * @throws DBALException
-     */
-    public function fillQueue(array $configuration, array $rootConfiguration = null)
+    public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
     {
         if ($rootConfiguration === null) {
             $rootConfiguration = $configuration;
@@ -56,8 +48,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
             ->join('r', self::TABLE_SYS_STORAGE, 's', 'r.uid_foreign = s.uid')
             ->where($storageQuery->expr()->eq('r.uid_local', (int)$configuration['uid']))
             ->getRestrictions()->removeAll()->add(GeneralUtility::makeInstance(FrontendRestrictionContainer::class));
-        if ($statement = $storageQuery->execute()) {
-            foreach ($statement as $storageRecord) {
+        if ($statement = $storageQuery->executeQuery()) {
+            foreach ($statement->fetchAllAssociative() as $storageRecord) {
                 $storages[] = $this->storageRepository->findByUid($storageRecord['storageId']);
             }
         }
@@ -103,10 +95,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
         return $result;
     }
 
-    /**
-     * @inheritDoc
-     */
-    public function processQueueItem(Item $item, array $configuration)
+    public function processQueueItem(Item $item, array $configuration): bool
     {
         $hash = md5($item->getIdentifier() . time());
         $queueManager = GeneralUtility::makeInstance(Manager::class);
@@ -118,7 +107,7 @@ public function processQueueItem(Item $item, array $configuration)
             [],
             [],
             1
-        )->fetch(\PDO::FETCH_ASSOC);
+        )->fetchAssociative();
         /** @var File $file */
         $file = GeneralUtility::makeInstance(FileRepository::class)->findByUid($item->getData()['fileId']);
 
@@ -142,35 +131,11 @@ public function processQueueItem(Item $item, array $configuration)
         return true;
     }
 
-    protected function getRootPage($pageId)
+    protected function getRootPage($pageId): int
     {
-        $pagesQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
-        $pagesQueryBuilder->select('uid', 'pid')->from('pages');
-        $pagesQueryBuilder->getRestrictions()
-            ->removeAll()
-            ->add(GeneralUtility::makeInstance(DeletedRestriction::class))
-            ->add(GeneralUtility::makeInstance(HiddenRestriction::class));
-        $domainQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_domain');
-        $domainQueryBuilder->getRestrictions()
-            ->removeAll()
-            ->add(GeneralUtility::makeInstance(HiddenRestriction::class));
-        $domainQueryBuilder->count('uid')->from('sys_domain');
-        do {
-            $domainQueryBuilder->where(
-                $domainQueryBuilder->expr()->eq('pid', $pageId)
-            );
-            $domainStatement = $domainQueryBuilder->execute();
-            if ($domainStatement->fetchColumn(0) > 0) {
-                return $pageId;
-            }
-
-            $pagesQueryBuilder->where(
-                $pagesQueryBuilder->expr()->eq('uid', $pageId)
-            );
-            $pagesStatement = $pagesQueryBuilder->execute();
-            $page = $pagesStatement->fetch(\PDO::FETCH_ASSOC);
-        } while ($page && $page['pid'] > 0 && $pageId = $page['pid']);
+        $siteFinder = GeneralUtility::makeInstance(SiteFinder::class);
+        $site = $siteFinder->getSiteByPageId($pageId);
 
-        return $pageId;
+        return $site->getRootPageId();
     }
 }
diff --git a/Classes/Crawler/FrontendRequestCrawler.php b/Classes/Crawler/FrontendRequestCrawler.php
@@ -3,6 +3,8 @@
 namespace WEBcoast\VersatileCrawler\Crawler;
 
 
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
 use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Site\Entity\Site;
@@ -17,8 +19,10 @@
 use WEBcoast\VersatileCrawler\Middleware\IndexingMiddleware;
 use WEBcoast\VersatileCrawler\Queue\Manager;
 
-abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface
+abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface, LoggerAwareInterface
 {
+    use LoggerAwareTrait;
+
     /**
      * @param \WEBcoast\VersatileCrawler\Domain\Model\Item                $item
      * @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController
@@ -59,12 +63,6 @@ public function processQueueItem(Item $item, array $configuration)
                 curl_setopt($curlHandle, CURLOPT_SSL_VERIFYHOST, false);
                 curl_setopt($curlHandle, CURLOPT_SSL_VERIFYPEER, false);
             }
-            // use this for debugging the frontend indexing part
-            if (isset($extensionConfiguration['xDebugForwardCookie']) && (int)$extensionConfiguration['xDebugForwardCookie'] === 1 && isset($_COOKIE['XDEBUG_SESSION'])) {
-                curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $_COOKIE['XDEBUG_SESSION']);
-            } elseif ($extensionConfiguration['xDebugSession'] && !empty($extensionConfiguration['xDebugSession'])) {
-                curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $extensionConfiguration['xDebugSession']);
-            }
             $content = curl_exec($curlHandle);
             $response = curl_getinfo($curlHandle);
             curl_close($curlHandle);
@@ -74,10 +72,12 @@ public function processQueueItem(Item $item, array $configuration)
             } else {
                 $result = json_decode($content, true);
                 $item->setState(Item::STATE_ERROR);
-                if (is_array($result)) {
+                if ($result['message'] ?? null) {
                     $item->setMessage($result['message']);
+                    $this->logger->error('Crawling failed: ' . $result['message'], ['result' => $result]);
                 } else {
                     $item->setMessage(sprintf('An error occurred. The call to the url "%s" returned the status code %d', $response['url'], $response['http_code']));
+                    $this->logger->error(sprintf('Crawling failed: The call to the url "%s" returned the status code %d', $response['url'], $response['http_code']), ['result' => $result]);
                 }
             }
         } catch (PageNotAvailableForIndexingException $exception) {

diff --git a/Classes/Crawler/Meta.php b/Classes/Crawler/Meta.php
@@ -11,14 +11,7 @@
 
 class Meta implements QueueInterface
 {
-
-    /**
-     * @param array $configuration
-     * @param array $rootConfiguration
-     *
-     * @return boolean
-     */
-    public function fillQueue(array $configuration, array $rootConfiguration = null)
+    public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
     {
         if ($rootConfiguration === null) {
             $rootConfiguration = $configuration;
@@ -36,8 +29,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
             $query->expr()->eq('m.uid_local', $configuration['uid'])
         );
         $result = true;
-        if ($statement = $query->execute()) {
-            foreach ($statement as $subConfiguration) {
+        if ($statement = $query->executeQuery()) {
+            foreach ($statement->fetchAllAssociative() as $subConfiguration) {
                 $className = TypeUtility::getClassForType($subConfiguration['type']);
                 $crawler = GeneralUtility::makeInstance($className);
                 if (!$crawler instanceof QueueInterface) {

diff --git a/Classes/Crawler/PageTree.php b/Classes/Crawler/PageTree.php
@@ -3,7 +3,6 @@
 namespace WEBcoast\VersatileCrawler\Crawler;
 
 
-use Doctrine\DBAL\DBALException;
 use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Context\Context;
 use TYPO3\CMS\Core\Context\VisibilityAspect;
@@ -39,15 +38,7 @@ public function __construct()
         }
     }
 
-    /**
-     * @param array      $configuration
-     * @param array|null $rootConfiguration
-     *
-     * @throws DBALException
-     *
-     * @return boolean
-     */
-    public function fillQueue(array $configuration, array $rootConfiguration = null)
+    public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
     {
         if ($rootConfiguration === null) {
             $rootConfiguration = $configuration;
@@ -80,7 +71,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
                             'uid!=' . $configuration['uid'],
                             'uid!=' . $rootConfiguration['uid']
                         );
-                    if ($query->execute()->fetchColumn(0) > 0) {
+                    if ($query->executeQuery()->fetchOne() > 0) {
                         continue;
                     }
                 }
@@ -132,7 +123,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
         return $result;
     }
 
-    protected function getPagesRecursively($page, &$pages, $level)
+    protected function getPagesRecursively($page, &$pages, $level): void
     {
         $subPages = $this->pageRepository->getMenu($page['uid']);
         foreach ($subPages as $subPage) {
@@ -144,14 +135,9 @@ protected function getPagesRecursively($page, &$pages, $level)
     }
 
     /**
-     * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
-     * @param array                                        $configuration
-     *
      * @throws PageNotAvailableForIndexingException
-     *
-     * @return string
      */
-    protected function buildQueryString(Item $item, array $configuration)
+    protected function buildQueryString(Item $item, array $configuration): string
     {
         $data = $item->getData();
         $page = $this->pageRepository->getPage($data['page']);
@@ -162,13 +148,7 @@ protected function buildQueryString(Item $item, array $configuration)
         return 'id=' . $data['page'] . '&L=' . $data['sys_language_uid'];
     }
 
-    /**
-     * @param \WEBcoast\VersatileCrawler\Domain\Model\Item                $item
-     * @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController
-     *
-     * @return boolean
-     */
-    public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController)
+    public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool
     {
         $data = $item->getData();
         return ($data['page'] === (int)$typoScriptFrontendController->id && $data['sys_language_uid'] === GeneralUtility::makeInstance(Context::class)->getAspect('language')->getId());

diff --git a/Classes/Crawler/Records.php b/Classes/Crawler/Records.php
@@ -33,13 +33,7 @@ public function __construct()
         $this->pageRepository = GeneralUtility::makeInstance(PageRepository::class);
     }
 
-    /**
-     * @param array $configuration
-     * @param array $rootConfiguration
-     *
-     * @return boolean
-     */
-    public function fillQueue(array $configuration, array $rootConfiguration = null)
+    public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
     {
         if ($rootConfiguration === null) {
             $rootConfiguration = $configuration;
@@ -78,19 +72,18 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
             );
         }
         $query->setRestrictions(GeneralUtility::makeInstance(FrontendRestrictionContainer::class));
-        // allow changing the query in sub classes
+        // allow changing the query in subclasses
         $this->alterQuery($query);
 
         $result = true;
         $context = GeneralUtility::makeInstance(Context::class);
-        $orginalVisibilityAspect = $context->getAspect('visibility');
+        $originalVisibilityAspect = $context->getAspect('visibility');
         $visibilityAspect = new VisibilityAspect(false, false, false);
         $context->setAspect('visibility', $visibilityAspect);
-        if ($statement = $query->execute()) {
-            $statement->setFetchMode(\PDO::FETCH_ASSOC);
+        if ($statement = $query->executeQuery()) {
             $queueManager = GeneralUtility::makeInstance(Manager::class);
             $languages = GeneralUtility::intExplode(',', $configuration['languages']);
-            foreach ($statement as $record) {
+            foreach ($statement->fetchAllAssociative() as $record) {
                 // no language field is set or the record is set to "all languages", index it in all languages
                 if (!isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']) || (int)$record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']] === -1) {
                     foreach ($languages as $language) {
@@ -181,12 +174,12 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
                 }
             }
         }
-        $context->setAspect('visibility', $orginalVisibilityAspect);
+        $context->setAspect('visibility', $originalVisibilityAspect);
 
         return $result;
     }
 
-    protected function getStoragePagesRecursively($pageUid, &$storagePages, $level)
+    protected function getStoragePagesRecursively($pageUid, &$storagePages, $level): void
     {
         if ($level === null || $level > 0) {
             $query = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
@@ -212,16 +205,17 @@ protected function getStoragePagesRecursively($pageUid, &$storagePages, $level)
     /**
      * @param QueryBuilder $query
      */
-    protected function alterQuery(&$query)
+    protected function alterQuery(&$query): void
     {
         // just do nothing here
     }
 
-    protected function addAdditionalItemData($record, &$data) {
+    protected function addAdditionalItemData($record, &$data): void
+    {
         // just do nothing here
     }
 
-    public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController)
+    public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool
     {
         $data = $item->getData();
         $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
@@ -261,7 +255,7 @@ public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typo
         return $isAllowed;
     }
 
-    protected function buildQueryString(Item $item, array $configuration)
+    protected function buildQueryString(Item $item, array $configuration): string
     {
 
         $data = $item->getData();
@@ -279,28 +273,23 @@ protected function buildQueryString(Item $item, array $configuration)
         return $queryString;
     }
 
-    /**
-     * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
-     *
-     * @return int
-     */
-    public function getRecordUid(Item $item)
+    public function getRecordUid(Item $item): int
     {
         $data = $item->getData();
 
         return ((int)$data['record_uid'] > 0 ? (int)$data['record_uid'] : 0);
     }
 
-    public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer)
+    public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer): void
     {
         $data = $item->getData();
         $configuration = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
             QueueController::CONFIGURATION_TABLE
-        )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetch();
+        )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetchAssociative();
         if (is_array($configuration) && isset($configuration['table_name'])) {
             $record = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
                 $configuration['table_name']
-            )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetch();
+            )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetchAssociative();
             if (is_array($record)) {
                 if (isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']) && $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']]) {
                     $indexer->conf['crdate'] = $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']];