diff --git a/Classes/Controller/CrawlerController.php b/Classes/Controller/CrawlerController.php index 5ccc99a..64e7fc4 100644 --- a/Classes/Controller/CrawlerController.php +++ b/Classes/Controller/CrawlerController.php @@ -53,7 +53,7 @@ public function processQueue($numberOfItemsPerRun) self::CONFIGURATION_TABLE, ['uid' => $itemArray['configuration']] ); - $configuration = $configurationResult->fetch(); + $configuration = $configurationResult->fetchAssociative(); $className = TypeUtility::getClassForType($configuration['type']); $crawler = GeneralUtility::makeInstance($className); if (!$crawler instanceof CrawlerInterface) { diff --git a/Classes/Controller/QueueController.php b/Classes/Controller/QueueController.php index b1d158e..f0d5ec8 100644 --- a/Classes/Controller/QueueController.php +++ b/Classes/Controller/QueueController.php @@ -41,7 +41,7 @@ public function fillQueue($configurations) $result = true; foreach ($configurations as $configurationUid) { $configurationResult = $connection->select(['*'], self::CONFIGURATION_TABLE, ['uid' => $configurationUid]); - $configuration = $configurationResult->fetch(); + $configuration = $configurationResult->fetchAssociative(); $className = TypeUtility::getClassForType($configuration['type']); $crawler = GeneralUtility::makeInstance($className); if (!$crawler instanceof QueueInterface) { @@ -60,4 +60,4 @@ public function fillQueue($configurations) return $result; } -} \ No newline at end of file +} diff --git a/Classes/Crawler/Files.php b/Classes/Crawler/Files.php index 9c3aa3f..49ac568 100644 --- a/Classes/Crawler/Files.php +++ b/Classes/Crawler/Files.php @@ -4,14 +4,13 @@ use Doctrine\DBAL\DBALException; use TYPO3\CMS\Core\Database\ConnectionPool; -use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction; use TYPO3\CMS\Core\Database\Query\Restriction\FrontendRestrictionContainer; -use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction; use TYPO3\CMS\Core\Resource\File; use TYPO3\CMS\Core\Resource\FileRepository; use TYPO3\CMS\Core\Resource\Folder; use TYPO3\CMS\Core\Resource\ResourceStorage; use TYPO3\CMS\Core\Resource\StorageRepository; +use TYPO3\CMS\Core\Site\SiteFinder; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\IndexedSearch\Indexer; use WEBcoast\VersatileCrawler\Controller\CrawlerController; @@ -35,14 +34,7 @@ public function __construct() $this->storageRepository = GeneralUtility::makeInstance(StorageRepository::class); } - /** - * @param array $configuration - * @param array $rootConfiguration - * - * @return boolean - * @throws DBALException - */ - public function fillQueue(array $configuration, array $rootConfiguration = null) + public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool { if ($rootConfiguration === null) { $rootConfiguration = $configuration; @@ -56,8 +48,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) ->join('r', self::TABLE_SYS_STORAGE, 's', 'r.uid_foreign = s.uid') ->where($storageQuery->expr()->eq('r.uid_local', (int)$configuration['uid'])) ->getRestrictions()->removeAll()->add(GeneralUtility::makeInstance(FrontendRestrictionContainer::class)); - if ($statement = $storageQuery->execute()) { - foreach ($statement as $storageRecord) { + if ($statement = $storageQuery->executeQuery()) { + foreach ($statement->fetchAllAssociative() as $storageRecord) { $storages[] = $this->storageRepository->findByUid($storageRecord['storageId']); } } @@ -103,10 +95,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) return $result; } - /** - * @inheritDoc - */ - public function processQueueItem(Item $item, array $configuration) + public function processQueueItem(Item $item, array $configuration): bool { $hash = md5($item->getIdentifier() . time()); $queueManager = GeneralUtility::makeInstance(Manager::class); @@ -118,7 +107,7 @@ public function processQueueItem(Item $item, array $configuration) [], [], 1 - )->fetch(\PDO::FETCH_ASSOC); + )->fetchAssociative(); /** @var File $file */ $file = GeneralUtility::makeInstance(FileRepository::class)->findByUid($item->getData()['fileId']); @@ -142,35 +131,11 @@ public function processQueueItem(Item $item, array $configuration) return true; } - protected function getRootPage($pageId) + protected function getRootPage($pageId): int { - $pagesQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages'); - $pagesQueryBuilder->select('uid', 'pid')->from('pages'); - $pagesQueryBuilder->getRestrictions() - ->removeAll() - ->add(GeneralUtility::makeInstance(DeletedRestriction::class)) - ->add(GeneralUtility::makeInstance(HiddenRestriction::class)); - $domainQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_domain'); - $domainQueryBuilder->getRestrictions() - ->removeAll() - ->add(GeneralUtility::makeInstance(HiddenRestriction::class)); - $domainQueryBuilder->count('uid')->from('sys_domain'); - do { - $domainQueryBuilder->where( - $domainQueryBuilder->expr()->eq('pid', $pageId) - ); - $domainStatement = $domainQueryBuilder->execute(); - if ($domainStatement->fetchColumn(0) > 0) { - return $pageId; - } - - $pagesQueryBuilder->where( - $pagesQueryBuilder->expr()->eq('uid', $pageId) - ); - $pagesStatement = $pagesQueryBuilder->execute(); - $page = $pagesStatement->fetch(\PDO::FETCH_ASSOC); - } while ($page && $page['pid'] > 0 && $pageId = $page['pid']); + $siteFinder = GeneralUtility::makeInstance(SiteFinder::class); + $site = $siteFinder->getSiteByPageId($pageId); - return $pageId; + return $site->getRootPageId(); } } diff --git a/Classes/Crawler/FrontendRequestCrawler.php b/Classes/Crawler/FrontendRequestCrawler.php index ad576ad..8afe14d 100644 --- a/Classes/Crawler/FrontendRequestCrawler.php +++ b/Classes/Crawler/FrontendRequestCrawler.php @@ -3,6 +3,8 @@ namespace WEBcoast\VersatileCrawler\Crawler; +use Psr\Log\LoggerAwareInterface; +use Psr\Log\LoggerAwareTrait; use TYPO3\CMS\Core\Configuration\ExtensionConfiguration; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Site\Entity\Site; @@ -17,8 +19,10 @@ use WEBcoast\VersatileCrawler\Middleware\IndexingMiddleware; use WEBcoast\VersatileCrawler\Queue\Manager; -abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface +abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface, LoggerAwareInterface { + use LoggerAwareTrait; + /** * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item * @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController @@ -59,12 +63,6 @@ public function processQueueItem(Item $item, array $configuration) curl_setopt($curlHandle, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($curlHandle, CURLOPT_SSL_VERIFYPEER, false); } - // use this for debugging the frontend indexing part - if (isset($extensionConfiguration['xDebugForwardCookie']) && (int)$extensionConfiguration['xDebugForwardCookie'] === 1 && isset($_COOKIE['XDEBUG_SESSION'])) { - curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $_COOKIE['XDEBUG_SESSION']); - } elseif ($extensionConfiguration['xDebugSession'] && !empty($extensionConfiguration['xDebugSession'])) { - curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $extensionConfiguration['xDebugSession']); - } $content = curl_exec($curlHandle); $response = curl_getinfo($curlHandle); curl_close($curlHandle); @@ -74,10 +72,12 @@ public function processQueueItem(Item $item, array $configuration) } else { $result = json_decode($content, true); $item->setState(Item::STATE_ERROR); - if (is_array($result)) { + if ($result['message'] ?? null) { $item->setMessage($result['message']); + $this->logger->error('Crawling failed: ' . $result['message'], ['result' => $result]); } else { $item->setMessage(sprintf('An error occurred. The call to the url "%s" returned the status code %d', $response['url'], $response['http_code'])); + $this->logger->error(sprintf('Crawling failed: The call to the url "%s" returned the status code %d', $response['url'], $response['http_code']), ['result' => $result]); } } } catch (PageNotAvailableForIndexingException $exception) { diff --git a/Classes/Crawler/Meta.php b/Classes/Crawler/Meta.php index 3e45e3e..8c59973 100644 --- a/Classes/Crawler/Meta.php +++ b/Classes/Crawler/Meta.php @@ -11,14 +11,7 @@ class Meta implements QueueInterface { - - /** - * @param array $configuration - * @param array $rootConfiguration - * - * @return boolean - */ - public function fillQueue(array $configuration, array $rootConfiguration = null) + public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool { if ($rootConfiguration === null) { $rootConfiguration = $configuration; @@ -36,8 +29,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) $query->expr()->eq('m.uid_local', $configuration['uid']) ); $result = true; - if ($statement = $query->execute()) { - foreach ($statement as $subConfiguration) { + if ($statement = $query->executeQuery()) { + foreach ($statement->fetchAllAssociative() as $subConfiguration) { $className = TypeUtility::getClassForType($subConfiguration['type']); $crawler = GeneralUtility::makeInstance($className); if (!$crawler instanceof QueueInterface) { diff --git a/Classes/Crawler/PageTree.php b/Classes/Crawler/PageTree.php index 4b8ec67..afff3e7 100644 --- a/Classes/Crawler/PageTree.php +++ b/Classes/Crawler/PageTree.php @@ -3,7 +3,6 @@ namespace WEBcoast\VersatileCrawler\Crawler; -use Doctrine\DBAL\DBALException; use TYPO3\CMS\Core\Configuration\ExtensionConfiguration; use TYPO3\CMS\Core\Context\Context; use TYPO3\CMS\Core\Context\VisibilityAspect; @@ -39,15 +38,7 @@ public function __construct() } } - /** - * @param array $configuration - * @param array|null $rootConfiguration - * - * @throws DBALException - * - * @return boolean - */ - public function fillQueue(array $configuration, array $rootConfiguration = null) + public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool { if ($rootConfiguration === null) { $rootConfiguration = $configuration; @@ -80,7 +71,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) 'uid!=' . $configuration['uid'], 'uid!=' . $rootConfiguration['uid'] ); - if ($query->execute()->fetchColumn(0) > 0) { + if ($query->executeQuery()->fetchOne() > 0) { continue; } } @@ -132,7 +123,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) return $result; } - protected function getPagesRecursively($page, &$pages, $level) + protected function getPagesRecursively($page, &$pages, $level): void { $subPages = $this->pageRepository->getMenu($page['uid']); foreach ($subPages as $subPage) { @@ -144,14 +135,9 @@ protected function getPagesRecursively($page, &$pages, $level) } /** - * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item - * @param array $configuration - * * @throws PageNotAvailableForIndexingException - * - * @return string */ - protected function buildQueryString(Item $item, array $configuration) + protected function buildQueryString(Item $item, array $configuration): string { $data = $item->getData(); $page = $this->pageRepository->getPage($data['page']); @@ -162,13 +148,7 @@ protected function buildQueryString(Item $item, array $configuration) return 'id=' . $data['page'] . '&L=' . $data['sys_language_uid']; } - /** - * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item - * @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController - * - * @return boolean - */ - public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController) + public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool { $data = $item->getData(); return ($data['page'] === (int)$typoScriptFrontendController->id && $data['sys_language_uid'] === GeneralUtility::makeInstance(Context::class)->getAspect('language')->getId()); diff --git a/Classes/Crawler/Records.php b/Classes/Crawler/Records.php index a678b02..45710c7 100644 --- a/Classes/Crawler/Records.php +++ b/Classes/Crawler/Records.php @@ -33,13 +33,7 @@ public function __construct() $this->pageRepository = GeneralUtility::makeInstance(PageRepository::class); } - /** - * @param array $configuration - * @param array $rootConfiguration - * - * @return boolean - */ - public function fillQueue(array $configuration, array $rootConfiguration = null) + public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool { if ($rootConfiguration === null) { $rootConfiguration = $configuration; @@ -78,19 +72,18 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) ); } $query->setRestrictions(GeneralUtility::makeInstance(FrontendRestrictionContainer::class)); - // allow changing the query in sub classes + // allow changing the query in subclasses $this->alterQuery($query); $result = true; $context = GeneralUtility::makeInstance(Context::class); - $orginalVisibilityAspect = $context->getAspect('visibility'); + $originalVisibilityAspect = $context->getAspect('visibility'); $visibilityAspect = new VisibilityAspect(false, false, false); $context->setAspect('visibility', $visibilityAspect); - if ($statement = $query->execute()) { - $statement->setFetchMode(\PDO::FETCH_ASSOC); + if ($statement = $query->executeQuery()) { $queueManager = GeneralUtility::makeInstance(Manager::class); $languages = GeneralUtility::intExplode(',', $configuration['languages']); - foreach ($statement as $record) { + foreach ($statement->fetchAllAssociative() as $record) { // no language field is set or the record is set to "all languages", index it in all languages if (!isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']) || (int)$record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']] === -1) { foreach ($languages as $language) { @@ -181,12 +174,12 @@ public function fillQueue(array $configuration, array $rootConfiguration = null) } } } - $context->setAspect('visibility', $orginalVisibilityAspect); + $context->setAspect('visibility', $originalVisibilityAspect); return $result; } - protected function getStoragePagesRecursively($pageUid, &$storagePages, $level) + protected function getStoragePagesRecursively($pageUid, &$storagePages, $level): void { if ($level === null || $level > 0) { $query = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages'); @@ -212,16 +205,17 @@ protected function getStoragePagesRecursively($pageUid, &$storagePages, $level) /** * @param QueryBuilder $query */ - protected function alterQuery(&$query) + protected function alterQuery(&$query): void { // just do nothing here } - protected function addAdditionalItemData($record, &$data) { + protected function addAdditionalItemData($record, &$data): void + { // just do nothing here } - public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController) + public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool { $data = $item->getData(); $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable( @@ -261,7 +255,7 @@ public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typo return $isAllowed; } - protected function buildQueryString(Item $item, array $configuration) + protected function buildQueryString(Item $item, array $configuration): string { $data = $item->getData(); @@ -279,28 +273,23 @@ protected function buildQueryString(Item $item, array $configuration) return $queryString; } - /** - * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item - * - * @return int - */ - public function getRecordUid(Item $item) + public function getRecordUid(Item $item): int { $data = $item->getData(); return ((int)$data['record_uid'] > 0 ? (int)$data['record_uid'] : 0); } - public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer) + public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer): void { $data = $item->getData(); $configuration = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable( QueueController::CONFIGURATION_TABLE - )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetch(); + )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetchAssociative(); if (is_array($configuration) && isset($configuration['table_name'])) { $record = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable( $configuration['table_name'] - )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetch(); + )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetchAssociative(); if (is_array($record)) { if (isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']) && $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']]) { $indexer->conf['crdate'] = $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']]; diff --git a/Classes/Domain/Model/Item.php b/Classes/Domain/Model/Item.php index d3bf6fa..fc68c3f 100644 --- a/Classes/Domain/Model/Item.php +++ b/Classes/Domain/Model/Item.php @@ -31,7 +31,7 @@ class Item extends AbstractEntity /** * @var array */ - protected $data; + protected array $data; /** * Item constructor. @@ -42,7 +42,7 @@ class Item extends AbstractEntity * @param string $message * @param array $data */ - public function __construct($configuration, $identifier, $state = self::STATE_PENDING, $message = '', $data = []) + public function __construct($configuration, $identifier, $state = self::STATE_PENDING, $message = '', array $data = []) { $this->configuration = $configuration; $this->identifier = $identifier; @@ -67,51 +67,36 @@ public function getIdentifier() return $this->identifier; } - /** - * @return mixed - */ - public function getState() + public function getState(): int { return $this->state; } - /** - * @return mixed - */ - public function getMessage() + public function setState(int $state): self { - return $this->message; + $this->state = $state; + return $this; } - /** - * @return mixed - */ - public function getData() + public function getMessage(): string { - return $this->data; + return $this->message; } - /** - * @param mixed $state - */ - public function setState($state) + public function setMessage(string $message): self { - $this->state = $state; + $this->message = $message; + return $this; } - /** - * @param mixed $message - */ - public function setMessage($message) + public function getData(): array { - $this->message = $message; + return $this->data; } - /** - * @param mixed $data - */ - public function setData($data) + public function setData(array $data): self { $this->data = $data; + return $this; } } diff --git a/Classes/Middleware/IndexingMiddleware.php b/Classes/Middleware/IndexingMiddleware.php index 333ad37..034d443 100644 --- a/Classes/Middleware/IndexingMiddleware.php +++ b/Classes/Middleware/IndexingMiddleware.php @@ -8,13 +8,11 @@ use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Server\MiddlewareInterface; use Psr\Http\Server\RequestHandlerInterface; -use TYPO3\CMS\Core\Context\Context; use TYPO3\CMS\Core\Context\LanguageAspect; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Http\JsonResponse; -use TYPO3\CMS\Core\Routing\PageArguments; +use TYPO3\CMS\Core\PageTitle\PageTitleProviderManager; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Core\Utility\VersionNumberUtility; use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; use TYPO3\CMS\IndexedSearch\Indexer; use WEBcoast\VersatileCrawler\Controller\CrawlerController; @@ -28,6 +26,20 @@ class IndexingMiddleware implements MiddlewareInterface { public const HASH_HEADER = 'X-Versatile-Crawler-Hash'; + protected PageTitleProviderManager $pageTitleProviderManager; + + protected Indexer $indexer; + + /** + * @param PageTitleProviderManager $pageTitleProviderManager + * @param Indexer $indexer + */ + public function __construct(PageTitleProviderManager $pageTitleProviderManager, Indexer $indexer) + { + $this->pageTitleProviderManager = $pageTitleProviderManager; + $this->indexer = $indexer; + } + public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface { if ($request->hasHeader(self::HASH_HEADER)) { @@ -47,8 +59,7 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface } $queueManager = GeneralUtility::makeInstance(Manager::class); - $itemResult = $queueManager->getItemForProcessing($hash); - $itemRecord = $itemResult->fetch(); + $itemRecord = $queueManager->getItemForProcessing($hash); if (!is_array($itemRecord)) { return new JsonResponse( [ @@ -110,7 +121,7 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface public function processIndexing(Item $item, TypoScriptFrontendController &$typoScriptFrontendController, FrontendRequestCrawler $crawler, ServerRequestInterface $request) { /** @var LanguageAspect $languageAspect */ - $languageAspect = GeneralUtility::makeInstance(Context::class)->getAspect('language'); + $languageAspect = $typoScriptFrontendController->getContext()->getAspect('language'); if ($languageAspect->getId() === $languageAspect->getContentId()) { $data = $item->getData(); $rootConfiguration = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(CrawlerController::CONFIGURATION_TABLE)->select( @@ -121,47 +132,35 @@ public function processIndexing(Item $item, TypoScriptFrontendController &$typoS [], 1 )->fetchAssociative(); - $indexer = GeneralUtility::makeInstance(Indexer::class); - $indexer->conf = []; - $indexer->conf['id'] = $typoScriptFrontendController->id; - $indexer->conf['type'] = $typoScriptFrontendController->type; - $indexer->conf['sys_language_uid'] = $languageAspect->getId(); - $indexer->conf['MP'] = $typoScriptFrontendController->MP; - $indexer->conf['gr_list'] = implode(',', GeneralUtility::makeInstance(Context::class)->getPropertyFromAspect('frontend.user', 'groupIds', [0, -1]));; - if (VersionNumberUtility::convertVersionNumberToInteger(VersionNumberUtility::getCurrentTypo3Version()) < VersionNumberUtility::convertVersionNumberToInteger('10.0.0')) { - // These two properties do not exist anymore in TYPO3 CMS 10 - $indexer->conf['cHash'] = $typoScriptFrontendController->cHash; - $indexer->conf['cHash_array'] = $typoScriptFrontendController->cHash_array; - } - $indexer->conf['staticPageArguments'] = []; - $pageArguments = $request->getAttribute('routing'); - if ($pageArguments instanceof PageArguments) { - $indexer->conf['staticPageArguments'] = $pageArguments->getStaticArguments(); - } - $indexer->conf['crdate'] = $typoScriptFrontendController->page['crdate']; - $indexer->conf['page_cache_reg1'] = $typoScriptFrontendController->page_cache_reg1; - $indexer->conf['rootline_uids'] = []; + $pageArguments = $typoScriptFrontendController->getPageArguments(); + $this->indexer->conf = []; + $this->indexer->conf['id'] = $typoScriptFrontendController->id; + $this->indexer->conf['type'] = $pageArguments->getPageType(); + $this->indexer->conf['sys_language_uid'] = $languageAspect->getId(); + $this->indexer->conf['MP'] = $typoScriptFrontendController->MP; + $this->indexer->conf['gr_list'] = implode(',', $typoScriptFrontendController->getContext()->getPropertyFromAspect('frontend.user', 'groupIds', [0, -1])); + $this->indexer->conf['staticPageArguments'] = $pageArguments->getStaticArguments(); + $this->indexer->conf['crdate'] = $typoScriptFrontendController->page['crdate']; + $this->indexer->conf['page_cache_reg1'] = $typoScriptFrontendController->page_cache_reg1; + $this->indexer->conf['rootline_uids'] = []; foreach ($typoScriptFrontendController->config['rootLine'] as $rlkey => $rldat) { - $indexer->conf['rootline_uids'][$rlkey] = $rldat['uid']; + $this->indexer->conf['rootline_uids'][$rlkey] = $rldat['uid']; } - $indexer->conf['content'] = $typoScriptFrontendController->content; - $indexer->conf['indexedDocTitle'] = $typoScriptFrontendController->convOutputCharset( - !empty($typoScriptFrontendController->altPageTitle) ? $typoScriptFrontendController->altPageTitle : $typoScriptFrontendController->indexedDocTitle - ); - $indexer->conf['metaCharset'] = $typoScriptFrontendController->metaCharset; - $indexer->conf['mtime'] = $typoScriptFrontendController->register['SYS_LASTCHANGED'] ?? $typoScriptFrontendController->page['SYS_LASTCHANGED']; - $indexer->conf['index_externals'] = $typoScriptFrontendController->config['config']['index_externals']; - $indexer->conf['index_descrLgd'] = $typoScriptFrontendController->config['config']['index_descrLgd']; - $indexer->conf['index_metatags'] = isset($typoScriptFrontendController->config['config']['index_metatags']) ?? true; - $indexer->conf['recordUid'] = $crawler->getRecordUid($item); - $indexer->conf['freeIndexUid'] = $rootConfiguration['indexing_configuration']; - $indexer->conf['freeIndexSetId'] = 0; + $this->indexer->conf['content'] = $typoScriptFrontendController->content; + $this->indexer->conf['indexedDocTitle'] = $this->pageTitleProviderManager->getTitle(); + $this->indexer->conf['mtime'] = $typoScriptFrontendController->register['SYS_LASTCHANGED'] ?? $typoScriptFrontendController->page['SYS_LASTCHANGED']; + $this->indexer->conf['index_externals'] = $typoScriptFrontendController->config['config']['index_externals'] ?? true; + $this->indexer->conf['index_descrLgd'] = $typoScriptFrontendController->config['config']['index_descrLgd'] ?? 0; + $this->indexer->conf['index_metatags'] = $typoScriptFrontendController->config['config']['index_metatags'] ?? true; + $this->indexer->conf['recordUid'] = $crawler->getRecordUid($item); + $this->indexer->conf['freeIndexUid'] = $rootConfiguration['indexing_configuration'] ?? 0; + $this->indexer->conf['freeIndexSetId'] = 0; // use this to override `crdate` and `mtime` and other information (used for record indexing) - $crawler->enrichIndexData($item, $typoScriptFrontendController, $indexer); + $crawler->enrichIndexData($item, $typoScriptFrontendController, $this->indexer); - $indexer->init(); - $indexer->indexTypo3PageContent(); + $this->indexer->init(); + $this->indexer->indexTypo3PageContent(); } } } diff --git a/Classes/Queue/Manager.php b/Classes/Queue/Manager.php index 90585f9..b1e931c 100644 --- a/Classes/Queue/Manager.php +++ b/Classes/Queue/Manager.php @@ -19,8 +19,6 @@ class Manager implements SingletonInterface * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item * * @return bool - * - * @throws \Doctrine\DBAL\DBALException */ public function addOrUpdateItem(Item $item) { @@ -33,7 +31,7 @@ public function addOrUpdateItem(Item $item) if ($count === 1) { $changedRows = $connection->update( self::QUEUE_TABLE, - ['tstamp' => time(), 'state' => Item::STATE_PENDING, 'message' => '', 'data' => json_encode($item->getData()), 'hash' => ''], + ['tstamp' => time(), 'state' => Item::STATE_PENDING, 'message' => '', 'data' => $item->getData(), 'hash' => ''], ['configuration' => $item->getConfiguration(), 'identifier' => $item->getIdentifier()] ); @@ -47,7 +45,7 @@ public function addOrUpdateItem(Item $item) 'tstamp' => time(), 'state' => Item::STATE_PENDING, 'message' => '', - 'data' => json_encode($item->getData()) + 'data' => $item->getData() ] ); @@ -62,7 +60,6 @@ public function addOrUpdateItem(Item $item) * * @return bool * - * @throws \Doctrine\DBAL\DBALException * @throws \RuntimeException */ public function updateState($item) @@ -91,7 +88,7 @@ public function cleanUpOldItems(int $timestamp, int $configuration) $query->where( $query->expr()->lt('tstamp', $timestamp), $query->expr()->eq('configuration', $configuration) - )->execute(); + )->executeStatement(); } /** @@ -112,7 +109,7 @@ public function getPendingItems($limit = null) [], ['tstamp' => 'ASC'], $limit !== null ? (int)$limit : 0 - ); + )->fetchAllAssociative(); } public function getAllItems() @@ -125,7 +122,7 @@ public function getAllItems() [], [], ['tstamp' => 'ASC'] - ); + )->fetchAllAssociative(); } public function getFinishedItems() @@ -133,14 +130,14 @@ public function getFinishedItems() $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); $query = $connection->createQueryBuilder()->select('*')->from(self::QUEUE_TABLE); $query->where( - $query->expr()->orX( + $query->expr()->or( 'state=' . Item::STATE_SUCCESS, 'state=' . Item::STATE_ERROR ) ); $query->orderBy('tstamp', 'ASC'); - return $query->execute(); + return $query->executeQuery()->fetchAllAssociative(); } public function getSuccessfulItems() @@ -169,12 +166,7 @@ public function getFailedItems() ); } - /** - * @return int - * - * @throws \Doctrine\DBAL\DBALException - */ - public function countAllItems() + public function countAllItems(): int { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); @@ -185,12 +177,7 @@ public function countAllItems() ); } - /** - * @return int - * - * @throws \Doctrine\DBAL\DBALException - */ - public function countFinishedItems() + public function countFinishedItems(): int { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); $query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE); @@ -201,10 +188,10 @@ public function countFinishedItems() ) ); - return $query->execute()->fetchColumn(0); + return $query->executeQuery()->fetchOne(); } - public function countSuccessfulItems() + public function countSuccessfulItems(): int { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); $query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE); @@ -212,10 +199,10 @@ public function countSuccessfulItems() $query->expr()->eq('state', Item::STATE_SUCCESS) ); - return $query->execute()->fetchColumn(0); + return $query->executeQuery()->fetchOne(); } - public function countFailedItems() + public function countFailedItems(): int { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); $query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE); @@ -223,7 +210,7 @@ public function countFailedItems() $query->expr()->eq('state', Item::STATE_ERROR) ); - return $query->execute()->fetchColumn(0); + return $query->executeQuery()->fetchOne(); } public function prepareItemForProcessing($configuration, $identifier, $hash) @@ -246,7 +233,7 @@ public function getItemForProcessing($hash) { $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE); - return $connection->select(['*'], self::QUEUE_TABLE, ['hash' => $hash, 'state' => Item::STATE_IN_PROGRESS]); + return $connection->select(['*'], self::QUEUE_TABLE, ['hash' => $hash, 'state' => Item::STATE_IN_PROGRESS])->fetchAssociative(); } public function removeQueueItem(Item $item) @@ -256,7 +243,7 @@ public function removeQueueItem(Item $item) return $connection->delete(self::QUEUE_TABLE, ['identifier' => $item->getIdentifier()]); } - public function getFromRecord($record) + public function getFromRecord($record): Item { return new Item( $record['configuration'], diff --git a/Classes/Scheduler/AbstractAdditionalFieldProvider.php b/Classes/Scheduler/AbstractAdditionalFieldProvider.php new file mode 100644 index 0000000..f220a39 --- /dev/null +++ b/Classes/Scheduler/AbstractAdditionalFieldProvider.php @@ -0,0 +1,15 @@ +countAllItems(); - $finishedItems = $queueManager->countFinishedItems(); - - if ($totalItems === 0) { - return 0; - } + $totalItems = $queueManager->countAllItems(); + $finishedItems = $queueManager->countFinishedItems(); - return $finishedItems / $totalItems * 100; - } catch (DBALException $e) { + if ($totalItems === 0) { return 0; } + + return $finishedItems / $totalItems * 100; } public function getAdditionalInformation() diff --git a/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php b/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php index 9f6a668..71b2495 100644 --- a/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php +++ b/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php @@ -4,16 +4,17 @@ use TYPO3\CMS\Core\Messaging\FlashMessage; -use TYPO3\CMS\Scheduler\AdditionalFieldProviderInterface; +use TYPO3\CMS\Scheduler\Task\AbstractTask; +use TYPO3\CMS\Scheduler\Task\Enumeration\Action; -class ProcessTaskAdditionalFieldProvider implements AdditionalFieldProviderInterface +class ProcessTaskAdditionalFieldProvider extends AbstractAdditionalFieldProvider { /** * Gets additional fields to render in the form to add/edit a task * * @param array $taskInfo Values of the fields from the * add/edit task form - * @param \TYPO3\CMS\Scheduler\Task\AbstractTask $task The task object being edited. + * @param AbstractTask $task The task object being edited. * Null when adding a task! * @param \TYPO3\CMS\Scheduler\Controller\SchedulerModuleController $schedulerModule Reference to the scheduler * backend module @@ -29,12 +30,12 @@ public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Schedule $fieldId = 'versatileCrawler_processTask_numberOfItemsPerRun'; $fieldName = 'tx_scheduler[' . $fieldId . ']'; - if ($schedulerModule->CMD === 'edit' && $task instanceof processTask && !isset($taskInfo[$fieldId])) { + if ($schedulerModule->getCurrentAction() == Action::EDIT && $task instanceof ProcessTask && !isset($taskInfo[$fieldId])) { $taskInfo[$fieldId] = $task->numberOfItemsPerRun; } $additionalFields[$fieldId] = [ - 'code' => '', + 'code' => '', 'label' => 'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.processTask.numberOfItemsPerRun' ]; @@ -59,7 +60,7 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched $submittedData['versatileCrawler_processTask_numberOfItemsPerRun'], '' ) !== 0 && (int)$submittedData['versatileCrawler_processTask_numberOfItemsPerRun'] < 1) { - $schedulerModule->addMessage( + $this->addMessage( $this->getLanguageService()->sL( 'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.processTask.error.numberOfItemsPerRun' ), @@ -76,18 +77,10 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched * * @param array $submittedData An array containing the data submitted by the * add/edit task form - * @param \TYPO3\CMS\Scheduler\Task\AbstractTask $task Reference to the scheduler backend module + * @param AbstractTask $task Reference to the scheduler backend module */ - public function saveAdditionalFields(array $submittedData, \TYPO3\CMS\Scheduler\Task\AbstractTask $task) + public function saveAdditionalFields(array $submittedData, AbstractTask $task) { $task->numberOfItemsPerRun = $submittedData['versatileCrawler_processTask_numberOfItemsPerRun']; } - - /** - * @return \TYPO3\CMS\Lang\LanguageService - */ - protected final function getLanguageService() - { - return $GLOBALS['LANG']; - } } diff --git a/Classes/Scheduler/QueueTask.php b/Classes/Scheduler/QueueTask.php index 8239a22..c071428 100644 --- a/Classes/Scheduler/QueueTask.php +++ b/Classes/Scheduler/QueueTask.php @@ -23,14 +23,14 @@ class QueueTask extends AbstractBaseTask * * @return bool Returns TRUE on successful execution, FALSE on error */ - public function execute() + public function execute(): bool { $queueController = GeneralUtility::makeInstance(QueueController::class); return $queueController->fillQueue($this->configurations); } - public function getAdditionalInformation() + public function getAdditionalInformation(): string { $configurationNames = []; @@ -43,7 +43,7 @@ public function getAdditionalInformation() QueueController::CONFIGURATION_TABLE, ['uid' => $configurationUid] ); - $configuration = $configurationResult->fetch(); + $configuration = $configurationResult->fetchAssociative(); if (is_array($configuration) && isset($configuration['title'])) { $configurationNames[] = $configuration['title']; } diff --git a/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php b/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php index 5e4408e..e92bebd 100644 --- a/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php +++ b/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php @@ -2,13 +2,13 @@ namespace WEBcoast\VersatileCrawler\Scheduler; - use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Messaging\FlashMessage; use TYPO3\CMS\Core\Utility\GeneralUtility; -use TYPO3\CMS\Scheduler\AdditionalFieldProviderInterface; +use TYPO3\CMS\Scheduler\Controller\SchedulerModuleController; +use TYPO3\CMS\Scheduler\Task\Enumeration\Action; -class QueueTaskAdditionalFieldProvider implements AdditionalFieldProviderInterface +class QueueTaskAdditionalFieldProvider extends AbstractAdditionalFieldProvider { const CONFIGURATION_TABLE = 'tx_versatilecrawler_domain_model_configuration'; @@ -25,7 +25,7 @@ class QueueTaskAdditionalFieldProvider implements AdditionalFieldProviderInterfa * @return array A two dimensional array, array('Identifier' => array('fieldId' => array('code' => '', 'label' => * '', 'cshKey' => '', 'cshLabel' => '')) */ - public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Scheduler\Controller\SchedulerModuleController $schedulerModule) + public function getAdditionalFields(array &$taskInfo, $task, SchedulerModuleController $schedulerModule) { $additionalFields = []; @@ -33,7 +33,7 @@ public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Schedule $fieldId = 'versatileCrawler_queueTask_configuration'; $fieldName = 'tx_scheduler[' . $fieldId . '][]'; - if ($schedulerModule->CMD === 'edit' && $task instanceof QueueTask && !isset($taskInfo[$fieldId])) { + if ($schedulerModule->getCurrentAction() == Action::EDIT && $task instanceof QueueTask && !isset($taskInfo[$fieldId])) { $taskInfo[$fieldId] = $task->configurations; } $fieldOptions = $this->getConfigurationOptions($taskInfo); @@ -61,7 +61,7 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched { $isValid = true; if (empty($submittedData['versatileCrawler_queueTask_configuration'])) { - $schedulerModule->addMessage( + $this->addMessage( $this->getLanguageService()->sL( 'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.queueTask.error.emptyConfiguration' ), @@ -95,20 +95,12 @@ protected function getConfigurationOptions($taskInfo) $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable( self::CONFIGURATION_TABLE ); - $configurations = $connection->select(['uid', 'title'], self::CONFIGURATION_TABLE, [], [], ['title' => 'ASC']); + $configurations = $connection->select(['uid', 'title'], self::CONFIGURATION_TABLE, [], [], ['title' => 'ASC'])->fetchAllAssociative(); $options = []; foreach ($configurations as $configuration) { - $options[] = ''; + $options[] = ''; } return implode('', $options); } - - /** - * @return \TYPO3\CMS\Lang\LanguageService - */ - protected final function getLanguageService() - { - return $GLOBALS['LANG']; - } } diff --git a/Classes/Tca/TableSelectItems.php b/Classes/Tca/TableSelectItems.php new file mode 100644 index 0000000..9abfc7f --- /dev/null +++ b/Classes/Tca/TableSelectItems.php @@ -0,0 +1,24 @@ + $configuration) { + if (in_array($table, ['pages', 'tt_content', 'sys_category']) || str_starts_with($table, 'tx_')) { + $items[] = [ + 'label' => $configuration['ctrl']['title'], + 'value' => $table + ]; + } + } + } +} diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml new file mode 100644 index 0000000..eb52adf --- /dev/null +++ b/Configuration/Services.yaml @@ -0,0 +1,8 @@ +services: + _defaults: + autowire: true + autoconfigure: true + public: false + + WEBcoast\VersatileCrawler\: + resource: '../Classes/*' diff --git a/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php b/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php index 174ae69..81862c8 100644 --- a/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php +++ b/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php @@ -101,7 +101,7 @@ 'config' => [ 'type' => 'select', 'renderType' => 'selectSingle', - 'special' => 'tables', + 'itemsProcFunc' => \WEBcoast\VersatileCrawler\Tca\TableSelectItems::class . '->addTableItems', ] ], 'record_storage_page' => [ diff --git a/composer.json b/composer.json index 7368c99..ef40cce 100644 --- a/composer.json +++ b/composer.json @@ -26,6 +26,7 @@ } }, "require": { - "typo3/cms-core": "^v10.4 || ^v11.5" + "php": "^8.1", + "typo3/cms-core": "^v12.4" } } diff --git a/ext_conf_template.txt b/ext_conf_template.txt index 48bda17..da172f8 100644 --- a/ext_conf_template.txt +++ b/ext_conf_template.txt @@ -3,9 +3,3 @@ disableCertificateCheck = 0 # cat=crawling; type=string; label=Page doktypes to crawl pagesDoktypes = 1 - -# cat=debugging; type=boolean; label=Forward XDebug session cookie -xDebugForwardCookie = 0 - -# cat=debugging; type=string; label=XDebug session -xDebugSession = diff --git a/ext_emconf.php b/ext_emconf.php index 08fe57a..c76412b 100644 --- a/ext_emconf.php +++ b/ext_emconf.php @@ -7,7 +7,7 @@ 'category' => 'plugin', 'constraints' => [ 'depends' => [ - 'typo3' => '10.4.0-11.5.99', + 'typo3' => '12.4.0-12.4.99', ], 'conflicts' => [ 'crawler' => '' diff --git a/ext_tables.php b/ext_tables.php index 93ab434..b2dffc3 100644 --- a/ext_tables.php +++ b/ext_tables.php @@ -1,14 +1,16 @@ registerIcon( - 'tcarecords-tx_versatilecrawler_domain_model_configuration-default', - TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class, - ['source' => 'EXT:versatile_crawler/Resources/Public/Icons/tx_versatilecrawler_domain_model_configuration.svg'] - ); +if (!defined('TYPO3')) { + die('Access denied.'); } + +\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::allowTableOnStandardPages( + 'tx_versatilecrawler_domain_model_configuration' +); + +$iconRegistry = TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(TYPO3\CMS\Core\Imaging\IconRegistry::class); +$iconRegistry->registerIcon( + 'tcarecords-tx_versatilecrawler_domain_model_configuration-default', + TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class, + ['source' => 'EXT:versatile_crawler/Resources/Public/Icons/tx_versatilecrawler_domain_model_configuration.svg'] +);