diff --git a/Classes/Controller/CrawlerController.php b/Classes/Controller/CrawlerController.php
index 5ccc99a..64e7fc4 100644
--- a/Classes/Controller/CrawlerController.php
+++ b/Classes/Controller/CrawlerController.php
@@ -53,7 +53,7 @@ public function processQueue($numberOfItemsPerRun)
self::CONFIGURATION_TABLE,
['uid' => $itemArray['configuration']]
);
- $configuration = $configurationResult->fetch();
+ $configuration = $configurationResult->fetchAssociative();
$className = TypeUtility::getClassForType($configuration['type']);
$crawler = GeneralUtility::makeInstance($className);
if (!$crawler instanceof CrawlerInterface) {
diff --git a/Classes/Controller/QueueController.php b/Classes/Controller/QueueController.php
index b1d158e..f0d5ec8 100644
--- a/Classes/Controller/QueueController.php
+++ b/Classes/Controller/QueueController.php
@@ -41,7 +41,7 @@ public function fillQueue($configurations)
$result = true;
foreach ($configurations as $configurationUid) {
$configurationResult = $connection->select(['*'], self::CONFIGURATION_TABLE, ['uid' => $configurationUid]);
- $configuration = $configurationResult->fetch();
+ $configuration = $configurationResult->fetchAssociative();
$className = TypeUtility::getClassForType($configuration['type']);
$crawler = GeneralUtility::makeInstance($className);
if (!$crawler instanceof QueueInterface) {
@@ -60,4 +60,4 @@ public function fillQueue($configurations)
return $result;
}
-}
\ No newline at end of file
+}
diff --git a/Classes/Crawler/Files.php b/Classes/Crawler/Files.php
index 9c3aa3f..49ac568 100644
--- a/Classes/Crawler/Files.php
+++ b/Classes/Crawler/Files.php
@@ -4,14 +4,13 @@
use Doctrine\DBAL\DBALException;
use TYPO3\CMS\Core\Database\ConnectionPool;
-use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Database\Query\Restriction\FrontendRestrictionContainer;
-use TYPO3\CMS\Core\Database\Query\Restriction\HiddenRestriction;
use TYPO3\CMS\Core\Resource\File;
use TYPO3\CMS\Core\Resource\FileRepository;
use TYPO3\CMS\Core\Resource\Folder;
use TYPO3\CMS\Core\Resource\ResourceStorage;
use TYPO3\CMS\Core\Resource\StorageRepository;
+use TYPO3\CMS\Core\Site\SiteFinder;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\IndexedSearch\Indexer;
use WEBcoast\VersatileCrawler\Controller\CrawlerController;
@@ -35,14 +34,7 @@ public function __construct()
$this->storageRepository = GeneralUtility::makeInstance(StorageRepository::class);
}
- /**
- * @param array $configuration
- * @param array $rootConfiguration
- *
- * @return boolean
- * @throws DBALException
- */
- public function fillQueue(array $configuration, array $rootConfiguration = null)
+ public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
{
if ($rootConfiguration === null) {
$rootConfiguration = $configuration;
@@ -56,8 +48,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
->join('r', self::TABLE_SYS_STORAGE, 's', 'r.uid_foreign = s.uid')
->where($storageQuery->expr()->eq('r.uid_local', (int)$configuration['uid']))
->getRestrictions()->removeAll()->add(GeneralUtility::makeInstance(FrontendRestrictionContainer::class));
- if ($statement = $storageQuery->execute()) {
- foreach ($statement as $storageRecord) {
+ if ($statement = $storageQuery->executeQuery()) {
+ foreach ($statement->fetchAllAssociative() as $storageRecord) {
$storages[] = $this->storageRepository->findByUid($storageRecord['storageId']);
}
}
@@ -103,10 +95,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
return $result;
}
- /**
- * @inheritDoc
- */
- public function processQueueItem(Item $item, array $configuration)
+ public function processQueueItem(Item $item, array $configuration): bool
{
$hash = md5($item->getIdentifier() . time());
$queueManager = GeneralUtility::makeInstance(Manager::class);
@@ -118,7 +107,7 @@ public function processQueueItem(Item $item, array $configuration)
[],
[],
1
- )->fetch(\PDO::FETCH_ASSOC);
+ )->fetchAssociative();
/** @var File $file */
$file = GeneralUtility::makeInstance(FileRepository::class)->findByUid($item->getData()['fileId']);
@@ -142,35 +131,11 @@ public function processQueueItem(Item $item, array $configuration)
return true;
}
- protected function getRootPage($pageId)
+ protected function getRootPage($pageId): int
{
- $pagesQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
- $pagesQueryBuilder->select('uid', 'pid')->from('pages');
- $pagesQueryBuilder->getRestrictions()
- ->removeAll()
- ->add(GeneralUtility::makeInstance(DeletedRestriction::class))
- ->add(GeneralUtility::makeInstance(HiddenRestriction::class));
- $domainQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_domain');
- $domainQueryBuilder->getRestrictions()
- ->removeAll()
- ->add(GeneralUtility::makeInstance(HiddenRestriction::class));
- $domainQueryBuilder->count('uid')->from('sys_domain');
- do {
- $domainQueryBuilder->where(
- $domainQueryBuilder->expr()->eq('pid', $pageId)
- );
- $domainStatement = $domainQueryBuilder->execute();
- if ($domainStatement->fetchColumn(0) > 0) {
- return $pageId;
- }
-
- $pagesQueryBuilder->where(
- $pagesQueryBuilder->expr()->eq('uid', $pageId)
- );
- $pagesStatement = $pagesQueryBuilder->execute();
- $page = $pagesStatement->fetch(\PDO::FETCH_ASSOC);
- } while ($page && $page['pid'] > 0 && $pageId = $page['pid']);
+ $siteFinder = GeneralUtility::makeInstance(SiteFinder::class);
+ $site = $siteFinder->getSiteByPageId($pageId);
- return $pageId;
+ return $site->getRootPageId();
}
}
diff --git a/Classes/Crawler/FrontendRequestCrawler.php b/Classes/Crawler/FrontendRequestCrawler.php
index ad576ad..8afe14d 100644
--- a/Classes/Crawler/FrontendRequestCrawler.php
+++ b/Classes/Crawler/FrontendRequestCrawler.php
@@ -3,6 +3,8 @@
namespace WEBcoast\VersatileCrawler\Crawler;
+use Psr\Log\LoggerAwareInterface;
+use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Site\Entity\Site;
@@ -17,8 +19,10 @@
use WEBcoast\VersatileCrawler\Middleware\IndexingMiddleware;
use WEBcoast\VersatileCrawler\Queue\Manager;
-abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface
+abstract class FrontendRequestCrawler implements CrawlerInterface, QueueInterface, LoggerAwareInterface
{
+ use LoggerAwareTrait;
+
/**
* @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
* @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController
@@ -59,12 +63,6 @@ public function processQueueItem(Item $item, array $configuration)
curl_setopt($curlHandle, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($curlHandle, CURLOPT_SSL_VERIFYPEER, false);
}
- // use this for debugging the frontend indexing part
- if (isset($extensionConfiguration['xDebugForwardCookie']) && (int)$extensionConfiguration['xDebugForwardCookie'] === 1 && isset($_COOKIE['XDEBUG_SESSION'])) {
- curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $_COOKIE['XDEBUG_SESSION']);
- } elseif ($extensionConfiguration['xDebugSession'] && !empty($extensionConfiguration['xDebugSession'])) {
- curl_setopt($curlHandle, CURLOPT_COOKIE, 'XDEBUG_SESSION=' . $extensionConfiguration['xDebugSession']);
- }
$content = curl_exec($curlHandle);
$response = curl_getinfo($curlHandle);
curl_close($curlHandle);
@@ -74,10 +72,12 @@ public function processQueueItem(Item $item, array $configuration)
} else {
$result = json_decode($content, true);
$item->setState(Item::STATE_ERROR);
- if (is_array($result)) {
+ if ($result['message'] ?? null) {
$item->setMessage($result['message']);
+ $this->logger->error('Crawling failed: ' . $result['message'], ['result' => $result]);
} else {
$item->setMessage(sprintf('An error occurred. The call to the url "%s" returned the status code %d', $response['url'], $response['http_code']));
+ $this->logger->error(sprintf('Crawling failed: The call to the url "%s" returned the status code %d', $response['url'], $response['http_code']), ['result' => $result]);
}
}
} catch (PageNotAvailableForIndexingException $exception) {
diff --git a/Classes/Crawler/Meta.php b/Classes/Crawler/Meta.php
index 3e45e3e..8c59973 100644
--- a/Classes/Crawler/Meta.php
+++ b/Classes/Crawler/Meta.php
@@ -11,14 +11,7 @@
class Meta implements QueueInterface
{
-
- /**
- * @param array $configuration
- * @param array $rootConfiguration
- *
- * @return boolean
- */
- public function fillQueue(array $configuration, array $rootConfiguration = null)
+ public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
{
if ($rootConfiguration === null) {
$rootConfiguration = $configuration;
@@ -36,8 +29,8 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
$query->expr()->eq('m.uid_local', $configuration['uid'])
);
$result = true;
- if ($statement = $query->execute()) {
- foreach ($statement as $subConfiguration) {
+ if ($statement = $query->executeQuery()) {
+ foreach ($statement->fetchAllAssociative() as $subConfiguration) {
$className = TypeUtility::getClassForType($subConfiguration['type']);
$crawler = GeneralUtility::makeInstance($className);
if (!$crawler instanceof QueueInterface) {
diff --git a/Classes/Crawler/PageTree.php b/Classes/Crawler/PageTree.php
index 4b8ec67..afff3e7 100644
--- a/Classes/Crawler/PageTree.php
+++ b/Classes/Crawler/PageTree.php
@@ -3,7 +3,6 @@
namespace WEBcoast\VersatileCrawler\Crawler;
-use Doctrine\DBAL\DBALException;
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Context\VisibilityAspect;
@@ -39,15 +38,7 @@ public function __construct()
}
}
- /**
- * @param array $configuration
- * @param array|null $rootConfiguration
- *
- * @throws DBALException
- *
- * @return boolean
- */
- public function fillQueue(array $configuration, array $rootConfiguration = null)
+ public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
{
if ($rootConfiguration === null) {
$rootConfiguration = $configuration;
@@ -80,7 +71,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
'uid!=' . $configuration['uid'],
'uid!=' . $rootConfiguration['uid']
);
- if ($query->execute()->fetchColumn(0) > 0) {
+ if ($query->executeQuery()->fetchOne() > 0) {
continue;
}
}
@@ -132,7 +123,7 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
return $result;
}
- protected function getPagesRecursively($page, &$pages, $level)
+ protected function getPagesRecursively($page, &$pages, $level): void
{
$subPages = $this->pageRepository->getMenu($page['uid']);
foreach ($subPages as $subPage) {
@@ -144,14 +135,9 @@ protected function getPagesRecursively($page, &$pages, $level)
}
/**
- * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
- * @param array $configuration
- *
* @throws PageNotAvailableForIndexingException
- *
- * @return string
*/
- protected function buildQueryString(Item $item, array $configuration)
+ protected function buildQueryString(Item $item, array $configuration): string
{
$data = $item->getData();
$page = $this->pageRepository->getPage($data['page']);
@@ -162,13 +148,7 @@ protected function buildQueryString(Item $item, array $configuration)
return 'id=' . $data['page'] . '&L=' . $data['sys_language_uid'];
}
- /**
- * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
- * @param \TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController $typoScriptFrontendController
- *
- * @return boolean
- */
- public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController)
+ public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool
{
$data = $item->getData();
return ($data['page'] === (int)$typoScriptFrontendController->id && $data['sys_language_uid'] === GeneralUtility::makeInstance(Context::class)->getAspect('language')->getId());
diff --git a/Classes/Crawler/Records.php b/Classes/Crawler/Records.php
index a678b02..45710c7 100644
--- a/Classes/Crawler/Records.php
+++ b/Classes/Crawler/Records.php
@@ -33,13 +33,7 @@ public function __construct()
$this->pageRepository = GeneralUtility::makeInstance(PageRepository::class);
}
- /**
- * @param array $configuration
- * @param array $rootConfiguration
- *
- * @return boolean
- */
- public function fillQueue(array $configuration, array $rootConfiguration = null)
+ public function fillQueue(array $configuration, ?array $rootConfiguration = null): bool
{
if ($rootConfiguration === null) {
$rootConfiguration = $configuration;
@@ -78,19 +72,18 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
);
}
$query->setRestrictions(GeneralUtility::makeInstance(FrontendRestrictionContainer::class));
- // allow changing the query in sub classes
+ // allow changing the query in subclasses
$this->alterQuery($query);
$result = true;
$context = GeneralUtility::makeInstance(Context::class);
- $orginalVisibilityAspect = $context->getAspect('visibility');
+ $originalVisibilityAspect = $context->getAspect('visibility');
$visibilityAspect = new VisibilityAspect(false, false, false);
$context->setAspect('visibility', $visibilityAspect);
- if ($statement = $query->execute()) {
- $statement->setFetchMode(\PDO::FETCH_ASSOC);
+ if ($statement = $query->executeQuery()) {
$queueManager = GeneralUtility::makeInstance(Manager::class);
$languages = GeneralUtility::intExplode(',', $configuration['languages']);
- foreach ($statement as $record) {
+ foreach ($statement->fetchAllAssociative() as $record) {
// no language field is set or the record is set to "all languages", index it in all languages
if (!isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']) || (int)$record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['languageField']] === -1) {
foreach ($languages as $language) {
@@ -181,12 +174,12 @@ public function fillQueue(array $configuration, array $rootConfiguration = null)
}
}
}
- $context->setAspect('visibility', $orginalVisibilityAspect);
+ $context->setAspect('visibility', $originalVisibilityAspect);
return $result;
}
- protected function getStoragePagesRecursively($pageUid, &$storagePages, $level)
+ protected function getStoragePagesRecursively($pageUid, &$storagePages, $level): void
{
if ($level === null || $level > 0) {
$query = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
@@ -212,16 +205,17 @@ protected function getStoragePagesRecursively($pageUid, &$storagePages, $level)
/**
* @param QueryBuilder $query
*/
- protected function alterQuery(&$query)
+ protected function alterQuery(&$query): void
{
// just do nothing here
}
- protected function addAdditionalItemData($record, &$data) {
+ protected function addAdditionalItemData($record, &$data): void
+ {
// just do nothing here
}
- public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController)
+ public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typoScriptFrontendController): bool
{
$data = $item->getData();
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
@@ -261,7 +255,7 @@ public function isIndexingAllowed(Item $item, TypoScriptFrontendController $typo
return $isAllowed;
}
- protected function buildQueryString(Item $item, array $configuration)
+ protected function buildQueryString(Item $item, array $configuration): string
{
$data = $item->getData();
@@ -279,28 +273,23 @@ protected function buildQueryString(Item $item, array $configuration)
return $queryString;
}
- /**
- * @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
- *
- * @return int
- */
- public function getRecordUid(Item $item)
+ public function getRecordUid(Item $item): int
{
$data = $item->getData();
return ((int)$data['record_uid'] > 0 ? (int)$data['record_uid'] : 0);
}
- public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer)
+ public function enrichIndexData(Item $item, TypoScriptFrontendController $typoScriptFrontendController, Indexer &$indexer): void
{
$data = $item->getData();
$configuration = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
QueueController::CONFIGURATION_TABLE
- )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetch();
+ )->select(['*'], QueueController::CONFIGURATION_TABLE, ['uid' => $item->getConfiguration()])->fetchAssociative();
if (is_array($configuration) && isset($configuration['table_name'])) {
$record = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
$configuration['table_name']
- )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetch();
+ )->select(['*'], $configuration['table_name'], ['uid' => $data['record_uid']])->fetchAssociative();
if (is_array($record)) {
if (isset($GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']) && $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']]) {
$indexer->conf['crdate'] = $record[$GLOBALS['TCA'][$configuration['table_name']]['ctrl']['crdate']];
diff --git a/Classes/Domain/Model/Item.php b/Classes/Domain/Model/Item.php
index d3bf6fa..fc68c3f 100644
--- a/Classes/Domain/Model/Item.php
+++ b/Classes/Domain/Model/Item.php
@@ -31,7 +31,7 @@ class Item extends AbstractEntity
/**
* @var array
*/
- protected $data;
+ protected array $data;
/**
* Item constructor.
@@ -42,7 +42,7 @@ class Item extends AbstractEntity
* @param string $message
* @param array $data
*/
- public function __construct($configuration, $identifier, $state = self::STATE_PENDING, $message = '', $data = [])
+ public function __construct($configuration, $identifier, $state = self::STATE_PENDING, $message = '', array $data = [])
{
$this->configuration = $configuration;
$this->identifier = $identifier;
@@ -67,51 +67,36 @@ public function getIdentifier()
return $this->identifier;
}
- /**
- * @return mixed
- */
- public function getState()
+ public function getState(): int
{
return $this->state;
}
- /**
- * @return mixed
- */
- public function getMessage()
+ public function setState(int $state): self
{
- return $this->message;
+ $this->state = $state;
+ return $this;
}
- /**
- * @return mixed
- */
- public function getData()
+ public function getMessage(): string
{
- return $this->data;
+ return $this->message;
}
- /**
- * @param mixed $state
- */
- public function setState($state)
+ public function setMessage(string $message): self
{
- $this->state = $state;
+ $this->message = $message;
+ return $this;
}
- /**
- * @param mixed $message
- */
- public function setMessage($message)
+ public function getData(): array
{
- $this->message = $message;
+ return $this->data;
}
- /**
- * @param mixed $data
- */
- public function setData($data)
+ public function setData(array $data): self
{
$this->data = $data;
+ return $this;
}
}
diff --git a/Classes/Middleware/IndexingMiddleware.php b/Classes/Middleware/IndexingMiddleware.php
index 333ad37..034d443 100644
--- a/Classes/Middleware/IndexingMiddleware.php
+++ b/Classes/Middleware/IndexingMiddleware.php
@@ -8,13 +8,11 @@
use Psr\Http\Message\ServerRequestInterface;
use Psr\Http\Server\MiddlewareInterface;
use Psr\Http\Server\RequestHandlerInterface;
-use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Context\LanguageAspect;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Http\JsonResponse;
-use TYPO3\CMS\Core\Routing\PageArguments;
+use TYPO3\CMS\Core\PageTitle\PageTitleProviderManager;
use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Core\Utility\VersionNumberUtility;
use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
use TYPO3\CMS\IndexedSearch\Indexer;
use WEBcoast\VersatileCrawler\Controller\CrawlerController;
@@ -28,6 +26,20 @@ class IndexingMiddleware implements MiddlewareInterface
{
public const HASH_HEADER = 'X-Versatile-Crawler-Hash';
+ protected PageTitleProviderManager $pageTitleProviderManager;
+
+ protected Indexer $indexer;
+
+ /**
+ * @param PageTitleProviderManager $pageTitleProviderManager
+ * @param Indexer $indexer
+ */
+ public function __construct(PageTitleProviderManager $pageTitleProviderManager, Indexer $indexer)
+ {
+ $this->pageTitleProviderManager = $pageTitleProviderManager;
+ $this->indexer = $indexer;
+ }
+
public function process(ServerRequestInterface $request, RequestHandlerInterface $handler): ResponseInterface
{
if ($request->hasHeader(self::HASH_HEADER)) {
@@ -47,8 +59,7 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface
}
$queueManager = GeneralUtility::makeInstance(Manager::class);
- $itemResult = $queueManager->getItemForProcessing($hash);
- $itemRecord = $itemResult->fetch();
+ $itemRecord = $queueManager->getItemForProcessing($hash);
if (!is_array($itemRecord)) {
return new JsonResponse(
[
@@ -110,7 +121,7 @@ public function process(ServerRequestInterface $request, RequestHandlerInterface
public function processIndexing(Item $item, TypoScriptFrontendController &$typoScriptFrontendController, FrontendRequestCrawler $crawler, ServerRequestInterface $request)
{
/** @var LanguageAspect $languageAspect */
- $languageAspect = GeneralUtility::makeInstance(Context::class)->getAspect('language');
+ $languageAspect = $typoScriptFrontendController->getContext()->getAspect('language');
if ($languageAspect->getId() === $languageAspect->getContentId()) {
$data = $item->getData();
$rootConfiguration = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(CrawlerController::CONFIGURATION_TABLE)->select(
@@ -121,47 +132,35 @@ public function processIndexing(Item $item, TypoScriptFrontendController &$typoS
[],
1
)->fetchAssociative();
- $indexer = GeneralUtility::makeInstance(Indexer::class);
- $indexer->conf = [];
- $indexer->conf['id'] = $typoScriptFrontendController->id;
- $indexer->conf['type'] = $typoScriptFrontendController->type;
- $indexer->conf['sys_language_uid'] = $languageAspect->getId();
- $indexer->conf['MP'] = $typoScriptFrontendController->MP;
- $indexer->conf['gr_list'] = implode(',', GeneralUtility::makeInstance(Context::class)->getPropertyFromAspect('frontend.user', 'groupIds', [0, -1]));;
- if (VersionNumberUtility::convertVersionNumberToInteger(VersionNumberUtility::getCurrentTypo3Version()) < VersionNumberUtility::convertVersionNumberToInteger('10.0.0')) {
- // These two properties do not exist anymore in TYPO3 CMS 10
- $indexer->conf['cHash'] = $typoScriptFrontendController->cHash;
- $indexer->conf['cHash_array'] = $typoScriptFrontendController->cHash_array;
- }
- $indexer->conf['staticPageArguments'] = [];
- $pageArguments = $request->getAttribute('routing');
- if ($pageArguments instanceof PageArguments) {
- $indexer->conf['staticPageArguments'] = $pageArguments->getStaticArguments();
- }
- $indexer->conf['crdate'] = $typoScriptFrontendController->page['crdate'];
- $indexer->conf['page_cache_reg1'] = $typoScriptFrontendController->page_cache_reg1;
- $indexer->conf['rootline_uids'] = [];
+ $pageArguments = $typoScriptFrontendController->getPageArguments();
+ $this->indexer->conf = [];
+ $this->indexer->conf['id'] = $typoScriptFrontendController->id;
+ $this->indexer->conf['type'] = $pageArguments->getPageType();
+ $this->indexer->conf['sys_language_uid'] = $languageAspect->getId();
+ $this->indexer->conf['MP'] = $typoScriptFrontendController->MP;
+ $this->indexer->conf['gr_list'] = implode(',', $typoScriptFrontendController->getContext()->getPropertyFromAspect('frontend.user', 'groupIds', [0, -1]));
+ $this->indexer->conf['staticPageArguments'] = $pageArguments->getStaticArguments();
+ $this->indexer->conf['crdate'] = $typoScriptFrontendController->page['crdate'];
+ $this->indexer->conf['page_cache_reg1'] = $typoScriptFrontendController->page_cache_reg1;
+ $this->indexer->conf['rootline_uids'] = [];
foreach ($typoScriptFrontendController->config['rootLine'] as $rlkey => $rldat) {
- $indexer->conf['rootline_uids'][$rlkey] = $rldat['uid'];
+ $this->indexer->conf['rootline_uids'][$rlkey] = $rldat['uid'];
}
- $indexer->conf['content'] = $typoScriptFrontendController->content;
- $indexer->conf['indexedDocTitle'] = $typoScriptFrontendController->convOutputCharset(
- !empty($typoScriptFrontendController->altPageTitle) ? $typoScriptFrontendController->altPageTitle : $typoScriptFrontendController->indexedDocTitle
- );
- $indexer->conf['metaCharset'] = $typoScriptFrontendController->metaCharset;
- $indexer->conf['mtime'] = $typoScriptFrontendController->register['SYS_LASTCHANGED'] ?? $typoScriptFrontendController->page['SYS_LASTCHANGED'];
- $indexer->conf['index_externals'] = $typoScriptFrontendController->config['config']['index_externals'];
- $indexer->conf['index_descrLgd'] = $typoScriptFrontendController->config['config']['index_descrLgd'];
- $indexer->conf['index_metatags'] = isset($typoScriptFrontendController->config['config']['index_metatags']) ?? true;
- $indexer->conf['recordUid'] = $crawler->getRecordUid($item);
- $indexer->conf['freeIndexUid'] = $rootConfiguration['indexing_configuration'];
- $indexer->conf['freeIndexSetId'] = 0;
+ $this->indexer->conf['content'] = $typoScriptFrontendController->content;
+ $this->indexer->conf['indexedDocTitle'] = $this->pageTitleProviderManager->getTitle();
+ $this->indexer->conf['mtime'] = $typoScriptFrontendController->register['SYS_LASTCHANGED'] ?? $typoScriptFrontendController->page['SYS_LASTCHANGED'];
+ $this->indexer->conf['index_externals'] = $typoScriptFrontendController->config['config']['index_externals'] ?? true;
+ $this->indexer->conf['index_descrLgd'] = $typoScriptFrontendController->config['config']['index_descrLgd'] ?? 0;
+ $this->indexer->conf['index_metatags'] = $typoScriptFrontendController->config['config']['index_metatags'] ?? true;
+ $this->indexer->conf['recordUid'] = $crawler->getRecordUid($item);
+ $this->indexer->conf['freeIndexUid'] = $rootConfiguration['indexing_configuration'] ?? 0;
+ $this->indexer->conf['freeIndexSetId'] = 0;
// use this to override `crdate` and `mtime` and other information (used for record indexing)
- $crawler->enrichIndexData($item, $typoScriptFrontendController, $indexer);
+ $crawler->enrichIndexData($item, $typoScriptFrontendController, $this->indexer);
- $indexer->init();
- $indexer->indexTypo3PageContent();
+ $this->indexer->init();
+ $this->indexer->indexTypo3PageContent();
}
}
}
diff --git a/Classes/Queue/Manager.php b/Classes/Queue/Manager.php
index 90585f9..b1e931c 100644
--- a/Classes/Queue/Manager.php
+++ b/Classes/Queue/Manager.php
@@ -19,8 +19,6 @@ class Manager implements SingletonInterface
* @param \WEBcoast\VersatileCrawler\Domain\Model\Item $item
*
* @return bool
- *
- * @throws \Doctrine\DBAL\DBALException
*/
public function addOrUpdateItem(Item $item)
{
@@ -33,7 +31,7 @@ public function addOrUpdateItem(Item $item)
if ($count === 1) {
$changedRows = $connection->update(
self::QUEUE_TABLE,
- ['tstamp' => time(), 'state' => Item::STATE_PENDING, 'message' => '', 'data' => json_encode($item->getData()), 'hash' => ''],
+ ['tstamp' => time(), 'state' => Item::STATE_PENDING, 'message' => '', 'data' => $item->getData(), 'hash' => ''],
['configuration' => $item->getConfiguration(), 'identifier' => $item->getIdentifier()]
);
@@ -47,7 +45,7 @@ public function addOrUpdateItem(Item $item)
'tstamp' => time(),
'state' => Item::STATE_PENDING,
'message' => '',
- 'data' => json_encode($item->getData())
+ 'data' => $item->getData()
]
);
@@ -62,7 +60,6 @@ public function addOrUpdateItem(Item $item)
*
* @return bool
*
- * @throws \Doctrine\DBAL\DBALException
* @throws \RuntimeException
*/
public function updateState($item)
@@ -91,7 +88,7 @@ public function cleanUpOldItems(int $timestamp, int $configuration)
$query->where(
$query->expr()->lt('tstamp', $timestamp),
$query->expr()->eq('configuration', $configuration)
- )->execute();
+ )->executeStatement();
}
/**
@@ -112,7 +109,7 @@ public function getPendingItems($limit = null)
[],
['tstamp' => 'ASC'],
$limit !== null ? (int)$limit : 0
- );
+ )->fetchAllAssociative();
}
public function getAllItems()
@@ -125,7 +122,7 @@ public function getAllItems()
[],
[],
['tstamp' => 'ASC']
- );
+ )->fetchAllAssociative();
}
public function getFinishedItems()
@@ -133,14 +130,14 @@ public function getFinishedItems()
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
$query = $connection->createQueryBuilder()->select('*')->from(self::QUEUE_TABLE);
$query->where(
- $query->expr()->orX(
+ $query->expr()->or(
'state=' . Item::STATE_SUCCESS,
'state=' . Item::STATE_ERROR
)
);
$query->orderBy('tstamp', 'ASC');
- return $query->execute();
+ return $query->executeQuery()->fetchAllAssociative();
}
public function getSuccessfulItems()
@@ -169,12 +166,7 @@ public function getFailedItems()
);
}
- /**
- * @return int
- *
- * @throws \Doctrine\DBAL\DBALException
- */
- public function countAllItems()
+ public function countAllItems(): int
{
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
@@ -185,12 +177,7 @@ public function countAllItems()
);
}
- /**
- * @return int
- *
- * @throws \Doctrine\DBAL\DBALException
- */
- public function countFinishedItems()
+ public function countFinishedItems(): int
{
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
$query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE);
@@ -201,10 +188,10 @@ public function countFinishedItems()
)
);
- return $query->execute()->fetchColumn(0);
+ return $query->executeQuery()->fetchOne();
}
- public function countSuccessfulItems()
+ public function countSuccessfulItems(): int
{
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
$query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE);
@@ -212,10 +199,10 @@ public function countSuccessfulItems()
$query->expr()->eq('state', Item::STATE_SUCCESS)
);
- return $query->execute()->fetchColumn(0);
+ return $query->executeQuery()->fetchOne();
}
- public function countFailedItems()
+ public function countFailedItems(): int
{
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
$query = $connection->createQueryBuilder()->count('*')->from(self::QUEUE_TABLE);
@@ -223,7 +210,7 @@ public function countFailedItems()
$query->expr()->eq('state', Item::STATE_ERROR)
);
- return $query->execute()->fetchColumn(0);
+ return $query->executeQuery()->fetchOne();
}
public function prepareItemForProcessing($configuration, $identifier, $hash)
@@ -246,7 +233,7 @@ public function getItemForProcessing($hash)
{
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::QUEUE_TABLE);
- return $connection->select(['*'], self::QUEUE_TABLE, ['hash' => $hash, 'state' => Item::STATE_IN_PROGRESS]);
+ return $connection->select(['*'], self::QUEUE_TABLE, ['hash' => $hash, 'state' => Item::STATE_IN_PROGRESS])->fetchAssociative();
}
public function removeQueueItem(Item $item)
@@ -256,7 +243,7 @@ public function removeQueueItem(Item $item)
return $connection->delete(self::QUEUE_TABLE, ['identifier' => $item->getIdentifier()]);
}
- public function getFromRecord($record)
+ public function getFromRecord($record): Item
{
return new Item(
$record['configuration'],
diff --git a/Classes/Scheduler/AbstractAdditionalFieldProvider.php b/Classes/Scheduler/AbstractAdditionalFieldProvider.php
new file mode 100644
index 0000000..f220a39
--- /dev/null
+++ b/Classes/Scheduler/AbstractAdditionalFieldProvider.php
@@ -0,0 +1,15 @@
+countAllItems();
- $finishedItems = $queueManager->countFinishedItems();
-
- if ($totalItems === 0) {
- return 0;
- }
+ $totalItems = $queueManager->countAllItems();
+ $finishedItems = $queueManager->countFinishedItems();
- return $finishedItems / $totalItems * 100;
- } catch (DBALException $e) {
+ if ($totalItems === 0) {
return 0;
}
+
+ return $finishedItems / $totalItems * 100;
}
public function getAdditionalInformation()
diff --git a/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php b/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php
index 9f6a668..71b2495 100644
--- a/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php
+++ b/Classes/Scheduler/ProcessTaskAdditionalFieldProvider.php
@@ -4,16 +4,17 @@
use TYPO3\CMS\Core\Messaging\FlashMessage;
-use TYPO3\CMS\Scheduler\AdditionalFieldProviderInterface;
+use TYPO3\CMS\Scheduler\Task\AbstractTask;
+use TYPO3\CMS\Scheduler\Task\Enumeration\Action;
-class ProcessTaskAdditionalFieldProvider implements AdditionalFieldProviderInterface
+class ProcessTaskAdditionalFieldProvider extends AbstractAdditionalFieldProvider
{
/**
* Gets additional fields to render in the form to add/edit a task
*
* @param array $taskInfo Values of the fields from the
* add/edit task form
- * @param \TYPO3\CMS\Scheduler\Task\AbstractTask $task The task object being edited.
+ * @param AbstractTask $task The task object being edited.
* Null when adding a task!
* @param \TYPO3\CMS\Scheduler\Controller\SchedulerModuleController $schedulerModule Reference to the scheduler
* backend module
@@ -29,12 +30,12 @@ public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Schedule
$fieldId = 'versatileCrawler_processTask_numberOfItemsPerRun';
$fieldName = 'tx_scheduler[' . $fieldId . ']';
- if ($schedulerModule->CMD === 'edit' && $task instanceof processTask && !isset($taskInfo[$fieldId])) {
+ if ($schedulerModule->getCurrentAction() == Action::EDIT && $task instanceof ProcessTask && !isset($taskInfo[$fieldId])) {
$taskInfo[$fieldId] = $task->numberOfItemsPerRun;
}
$additionalFields[$fieldId] = [
- 'code' => '',
+ 'code' => '',
'label' => 'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.processTask.numberOfItemsPerRun'
];
@@ -59,7 +60,7 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched
$submittedData['versatileCrawler_processTask_numberOfItemsPerRun'],
''
) !== 0 && (int)$submittedData['versatileCrawler_processTask_numberOfItemsPerRun'] < 1) {
- $schedulerModule->addMessage(
+ $this->addMessage(
$this->getLanguageService()->sL(
'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.processTask.error.numberOfItemsPerRun'
),
@@ -76,18 +77,10 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched
*
* @param array $submittedData An array containing the data submitted by the
* add/edit task form
- * @param \TYPO3\CMS\Scheduler\Task\AbstractTask $task Reference to the scheduler backend module
+ * @param AbstractTask $task Reference to the scheduler backend module
*/
- public function saveAdditionalFields(array $submittedData, \TYPO3\CMS\Scheduler\Task\AbstractTask $task)
+ public function saveAdditionalFields(array $submittedData, AbstractTask $task)
{
$task->numberOfItemsPerRun = $submittedData['versatileCrawler_processTask_numberOfItemsPerRun'];
}
-
- /**
- * @return \TYPO3\CMS\Lang\LanguageService
- */
- protected final function getLanguageService()
- {
- return $GLOBALS['LANG'];
- }
}
diff --git a/Classes/Scheduler/QueueTask.php b/Classes/Scheduler/QueueTask.php
index 8239a22..c071428 100644
--- a/Classes/Scheduler/QueueTask.php
+++ b/Classes/Scheduler/QueueTask.php
@@ -23,14 +23,14 @@ class QueueTask extends AbstractBaseTask
*
* @return bool Returns TRUE on successful execution, FALSE on error
*/
- public function execute()
+ public function execute(): bool
{
$queueController = GeneralUtility::makeInstance(QueueController::class);
return $queueController->fillQueue($this->configurations);
}
- public function getAdditionalInformation()
+ public function getAdditionalInformation(): string
{
$configurationNames = [];
@@ -43,7 +43,7 @@ public function getAdditionalInformation()
QueueController::CONFIGURATION_TABLE,
['uid' => $configurationUid]
);
- $configuration = $configurationResult->fetch();
+ $configuration = $configurationResult->fetchAssociative();
if (is_array($configuration) && isset($configuration['title'])) {
$configurationNames[] = $configuration['title'];
}
diff --git a/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php b/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php
index 5e4408e..e92bebd 100644
--- a/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php
+++ b/Classes/Scheduler/QueueTaskAdditionalFieldProvider.php
@@ -2,13 +2,13 @@
namespace WEBcoast\VersatileCrawler\Scheduler;
-
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Messaging\FlashMessage;
use TYPO3\CMS\Core\Utility\GeneralUtility;
-use TYPO3\CMS\Scheduler\AdditionalFieldProviderInterface;
+use TYPO3\CMS\Scheduler\Controller\SchedulerModuleController;
+use TYPO3\CMS\Scheduler\Task\Enumeration\Action;
-class QueueTaskAdditionalFieldProvider implements AdditionalFieldProviderInterface
+class QueueTaskAdditionalFieldProvider extends AbstractAdditionalFieldProvider
{
const CONFIGURATION_TABLE = 'tx_versatilecrawler_domain_model_configuration';
@@ -25,7 +25,7 @@ class QueueTaskAdditionalFieldProvider implements AdditionalFieldProviderInterfa
* @return array A two dimensional array, array('Identifier' => array('fieldId' => array('code' => '', 'label' =>
* '', 'cshKey' => '', 'cshLabel' => ''))
*/
- public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Scheduler\Controller\SchedulerModuleController $schedulerModule)
+ public function getAdditionalFields(array &$taskInfo, $task, SchedulerModuleController $schedulerModule)
{
$additionalFields = [];
@@ -33,7 +33,7 @@ public function getAdditionalFields(array &$taskInfo, $task, \TYPO3\CMS\Schedule
$fieldId = 'versatileCrawler_queueTask_configuration';
$fieldName = 'tx_scheduler[' . $fieldId . '][]';
- if ($schedulerModule->CMD === 'edit' && $task instanceof QueueTask && !isset($taskInfo[$fieldId])) {
+ if ($schedulerModule->getCurrentAction() == Action::EDIT && $task instanceof QueueTask && !isset($taskInfo[$fieldId])) {
$taskInfo[$fieldId] = $task->configurations;
}
$fieldOptions = $this->getConfigurationOptions($taskInfo);
@@ -61,7 +61,7 @@ public function validateAdditionalFields(array &$submittedData, \TYPO3\CMS\Sched
{
$isValid = true;
if (empty($submittedData['versatileCrawler_queueTask_configuration'])) {
- $schedulerModule->addMessage(
+ $this->addMessage(
$this->getLanguageService()->sL(
'LLL:EXT:versatile_crawler/Resources/Private/Language/locallang_backend.xlf:scheduler.queueTask.error.emptyConfiguration'
),
@@ -95,20 +95,12 @@ protected function getConfigurationOptions($taskInfo)
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(
self::CONFIGURATION_TABLE
);
- $configurations = $connection->select(['uid', 'title'], self::CONFIGURATION_TABLE, [], [], ['title' => 'ASC']);
+ $configurations = $connection->select(['uid', 'title'], self::CONFIGURATION_TABLE, [], [], ['title' => 'ASC'])->fetchAllAssociative();
$options = [];
foreach ($configurations as $configuration) {
- $options[] = '';
+ $options[] = '';
}
return implode('', $options);
}
-
- /**
- * @return \TYPO3\CMS\Lang\LanguageService
- */
- protected final function getLanguageService()
- {
- return $GLOBALS['LANG'];
- }
}
diff --git a/Classes/Tca/TableSelectItems.php b/Classes/Tca/TableSelectItems.php
new file mode 100644
index 0000000..9abfc7f
--- /dev/null
+++ b/Classes/Tca/TableSelectItems.php
@@ -0,0 +1,24 @@
+ $configuration) {
+ if (in_array($table, ['pages', 'tt_content', 'sys_category']) || str_starts_with($table, 'tx_')) {
+ $items[] = [
+ 'label' => $configuration['ctrl']['title'],
+ 'value' => $table
+ ];
+ }
+ }
+ }
+}
diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml
new file mode 100644
index 0000000..eb52adf
--- /dev/null
+++ b/Configuration/Services.yaml
@@ -0,0 +1,8 @@
+services:
+ _defaults:
+ autowire: true
+ autoconfigure: true
+ public: false
+
+ WEBcoast\VersatileCrawler\:
+ resource: '../Classes/*'
diff --git a/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php b/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php
index 174ae69..81862c8 100644
--- a/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php
+++ b/Configuration/TCA/tx_versatilecrawler_domain_model_configuration.php
@@ -101,7 +101,7 @@
'config' => [
'type' => 'select',
'renderType' => 'selectSingle',
- 'special' => 'tables',
+ 'itemsProcFunc' => \WEBcoast\VersatileCrawler\Tca\TableSelectItems::class . '->addTableItems',
]
],
'record_storage_page' => [
diff --git a/composer.json b/composer.json
index 7368c99..ef40cce 100644
--- a/composer.json
+++ b/composer.json
@@ -26,6 +26,7 @@
}
},
"require": {
- "typo3/cms-core": "^v10.4 || ^v11.5"
+ "php": "^8.1",
+ "typo3/cms-core": "^v12.4"
}
}
diff --git a/ext_conf_template.txt b/ext_conf_template.txt
index 48bda17..da172f8 100644
--- a/ext_conf_template.txt
+++ b/ext_conf_template.txt
@@ -3,9 +3,3 @@ disableCertificateCheck = 0
# cat=crawling; type=string; label=Page doktypes to crawl
pagesDoktypes = 1
-
-# cat=debugging; type=boolean; label=Forward XDebug session cookie
-xDebugForwardCookie = 0
-
-# cat=debugging; type=string; label=XDebug session
-xDebugSession =
diff --git a/ext_emconf.php b/ext_emconf.php
index 08fe57a..c76412b 100644
--- a/ext_emconf.php
+++ b/ext_emconf.php
@@ -7,7 +7,7 @@
'category' => 'plugin',
'constraints' => [
'depends' => [
- 'typo3' => '10.4.0-11.5.99',
+ 'typo3' => '12.4.0-12.4.99',
],
'conflicts' => [
'crawler' => ''
diff --git a/ext_tables.php b/ext_tables.php
index 93ab434..b2dffc3 100644
--- a/ext_tables.php
+++ b/ext_tables.php
@@ -1,14 +1,16 @@
registerIcon(
- 'tcarecords-tx_versatilecrawler_domain_model_configuration-default',
- TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class,
- ['source' => 'EXT:versatile_crawler/Resources/Public/Icons/tx_versatilecrawler_domain_model_configuration.svg']
- );
+if (!defined('TYPO3')) {
+ die('Access denied.');
}
+
+\TYPO3\CMS\Core\Utility\ExtensionManagementUtility::allowTableOnStandardPages(
+ 'tx_versatilecrawler_domain_model_configuration'
+);
+
+$iconRegistry = TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(TYPO3\CMS\Core\Imaging\IconRegistry::class);
+$iconRegistry->registerIcon(
+ 'tcarecords-tx_versatilecrawler_domain_model_configuration-default',
+ TYPO3\CMS\Core\Imaging\IconProvider\SvgIconProvider::class,
+ ['source' => 'EXT:versatile_crawler/Resources/Public/Icons/tx_versatilecrawler_domain_model_configuration.svg']
+);