From 51133188b84ff538f2e3552a377f246c4eadbbea Mon Sep 17 00:00:00 2001 From: Josh Crawford Date: Wed, 22 Jun 2016 00:05:20 +1000 Subject: [PATCH] rework delete handling Now updates all existing entries, and deletes any not found in the feed. --- feedme/services/FeedMeService.php | 83 ++++++++++++++++++++----------- feedme/tasks/FeedMeTask.php | 14 +++++- feedme/templates/feeds/_edit.html | 6 +-- 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/feedme/services/FeedMeService.php b/feedme/services/FeedMeService.php index 3023b99a..61e90daa 100644 --- a/feedme/services/FeedMeService.php +++ b/feedme/services/FeedMeService.php @@ -8,46 +8,38 @@ class FeedMeService extends BaseApplicationComponent public function setupForImport($feed) { - // Return a collection of only the fields we're mapping - $fields = array(); + $return = array( + 'fields' => array(), + 'processedEntries' => array(), + 'existingEntries' => array(), + ); // Start looping through all the mapped fields - checking for nested nodes foreach ($feed['fieldMapping'] as $itemNode => $destination) { // Forget about any fields mapped as not to import if ($destination != 'noimport') { - $fields[$itemNode] = $destination; + $return['fields'][$itemNode] = $destination; } } // // If our duplication handling is to delete - we delete all entries in this section/entrytype // - if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete) { - try { - - $criteria = craft()->feedMe_entry->setCriteria($feed); - $entries = $criteria->find(); - - if (!craft()->entries->deleteEntry($entries)) { - FeedMePlugin::log('FeedMeError - Something went wrong while deleting entries.', LogLevel::Error, true); - } - } catch (\Exception $e) { - FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true); - } + $criteria = craft()->feedMe_entry->setCriteria($feed); + $return['existingEntries'] = $criteria->ids(); } // // Return variables that need to be used per-node, but only need to be processed once. // - return array( - 'fields' => $fields, - ); + return $return; } public function importNode($nodes, $feed, $settings) { + $processedEntries = array(); $hasAnyErrors = false; $time_start = microtime(true); @@ -56,8 +48,12 @@ public function importNode($nodes, $feed, $settings) foreach ($nodes as $key => $node) { $result = $this->importSingleNode($node, $feed, $settings); + if (isset($result['entryId'])) { + $processedEntries[] = $result['entryId']; + } + // Report back if even one feed node failed - if (!$result) { + if (!$result['result']) { $hasAnyErrors = true; } } @@ -66,7 +62,7 @@ public function importNode($nodes, $feed, $settings) $execution_time = number_format(($time_end - $time_start), 2); FeedMePlugin::log($feed->name . ': Processing finished in ' . $execution_time . 's', LogLevel::Info, true); - return !$hasAnyErrors; + return array('result' => !$hasAnyErrors, 'processedEntries' => $processedEntries); } public function importSingleNode($node, $feed, $settings) @@ -121,7 +117,7 @@ public function importSingleNode($node, $feed, $settings) } catch (\Exception $e) { FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true); - return false; + return array('result' => false); } } @@ -137,16 +133,24 @@ public function importSingleNode($node, $feed, $settings) // If there's an existing matching entry - if ($existingEntry && $feed['duplicateHandle'] != FeedMe_Duplicate::Delete) { + if ($existingEntry && $feed['duplicateHandle']) { + + // If we're deleting + if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete) { + + // Fill new EntryModel with match + $entry = $existingEntry; + } // If we're updating if ($feed['duplicateHandle'] == FeedMe_Duplicate::Update) { // Fill new EntryModel with match $entry = $existingEntry; + } // If we're adding, make sure not to overwrite existing entry - } else if ($feed['duplicateHandle'] == FeedMe_Duplicate::Add) { + if ($feed['duplicateHandle'] == FeedMe_Duplicate::Add) { $canSaveEntry = false; } } else { @@ -179,7 +183,7 @@ public function importSingleNode($node, $feed, $settings) if (!craft()->entries->saveEntry($entry)) { FeedMePlugin::log($feed->name . ': ' . json_encode($entry->getErrors()), LogLevel::Error, true); - return false; + return array('result' => false); } else { // If we're importing into a specific locale, we need to create this entry if it doesn't already exist @@ -193,7 +197,7 @@ public function importSingleNode($node, $feed, $settings) if (!craft()->entries->saveEntry($entryLocale)) { FeedMePlugin::log($feed->name . ': ' . json_encode($entryLocale->getErrors()), LogLevel::Error, true); - return false; + return array('result' => false); } else { // Successfully saved/added entry @@ -203,7 +207,7 @@ public function importSingleNode($node, $feed, $settings) FeedMePlugin::log($feed->name . ': Entry successfully added: ' . $entryLocale->id, LogLevel::Info, true); } - return true; + return array('result' => true, 'entryId' => $entryLocale->id); } } else { @@ -214,20 +218,41 @@ public function importSingleNode($node, $feed, $settings) FeedMePlugin::log($feed->name . ': Entry successfully added: ' . $entry->id, LogLevel::Info, true); } - return true; + return array('result' => true, 'entryId' => $entry->id); } } } catch (\Exception $e) { FeedMePlugin::log($feed->name . ': Entry FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true); - return false; + return array('result' => false, 'entryId' => $entry->id); } } else { if ($existingEntry) { FeedMePlugin::log($feed->name . ': Entry skipped: ' . $existingEntry->id . '.', LogLevel::Error, true); } - return true; + return array('result' => true); + } + } + + public function deleteLeftoverEntries($settings, $feed, $processedEntries, $result) + { + if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete && $result['result']) { + $deleteIds = array_diff($settings['existingEntries'], $processedEntries); + + $criteria = craft()->feedMe_entry->setCriteria($feed); + $criteria->id = $deleteIds; + $entriesToDelete = $criteria->find(); + + try { + if (!craft()->entries->deleteEntry($entriesToDelete)) { + FeedMePlugin::log('FeedMeError - Something went wrong while deleting entries.', LogLevel::Error, true); + } else { + FeedMePlugin::log($feed->name . ': The following entries have been deleted: ' . print_r($deleteIds, true) . '.', LogLevel::Error, true); + } + } catch (\Exception $e) { + FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true); + } } } } diff --git a/feedme/tasks/FeedMeTask.php b/feedme/tasks/FeedMeTask.php index 53c36473..5cdec486 100644 --- a/feedme/tasks/FeedMeTask.php +++ b/feedme/tasks/FeedMeTask.php @@ -12,6 +12,7 @@ class FeedMeTask extends BaseTask private $_feedSettings; private $_backup; private $_chunkedFeedData; + private $_processedEntries = array(); // Public Methods // ========================================================================= @@ -35,6 +36,11 @@ public function getTotalSteps() // Get the data for the mapping screen, based on the URL provided $this->_feedData = craft()->feedMe_feed->getFeed($this->_feed->feedType, $this->_feed->feedUrl, $this->_feed->primaryElement); + if (!$this->_feedData) { + FeedMePlugin::log($this->_feed->name . ': FeedMeError', LogLevel::Error, true); + return false; + } + // Chunk the feed data into chunks of 100 - optimises mapping process by not calling service each step $this->_chunkedFeedData = array_chunk($this->_feedData, 100); @@ -53,8 +59,14 @@ public function getTotalSteps() public function runStep($step) { $result = craft()->feedMe->importNode($this->_chunkedFeedData[$step], $this->_feed, $this->_feedSettings); + $this->_processedEntries = array_merge($this->_processedEntries, $result['processedEntries']); + + // For delete, at the end of our processing, we delete all entries not recorded + if ($step == $this->getTotalSteps()-1) { + craft()->feedMe->deleteLeftoverEntries($this->_feedSettings, $this->_feed, $this->_processedEntries, $result); + } - if (!$result) { + if (!$result['result']) { return 'Feed Me Failure: Check Feed Me logs.'; } else { return true; diff --git a/feedme/templates/feeds/_edit.html b/feedme/templates/feeds/_edit.html index a5452179..ccd2dfa4 100644 --- a/feedme/templates/feeds/_edit.html +++ b/feedme/templates/feeds/_edit.html @@ -117,11 +117,11 @@
{{ forms.selectField({ - label: "Import strategy Add Entries: Duplicate entries will be skipped, new entries however, will be added. \"I want to keep existing entries untouched but add new ones.\"Update Entries: Select fields to compare duplicate entries with, updating all other fields.\"I want to update existing entries and add new ones.\"Delete Entries: Delete all existing entries in this section, adding only entries from this feed. Be careful.\"I want only the entries from this feed in this section.\"" | t, - instructions: 'Choose how to handle duplicate records.', + label: "Import strategy Add Unique Only: Duplicate entries will be skipped, new entries however, will be added. \"I want to keep existing entries untouched but add new ones.\"Add and Update existing: Select fields to compare duplicate entries with, updating all other fields.\"I want to update existing entries and add new ones.\"Delete Entries existing: Update any existing entries, and delete any entries not updated by this feed. Be careful.\"I want only the entries from this feed in this section.\"" | t, + instructions: 'Choose what happens when this feed is run again. Click tooltip above for explanation.', id: 'duplicateHandle', name: 'duplicateHandle', - options: {'add': 'Add Unique Entries Only', 'update': 'Add and Update Entries', 'delete': 'Delete all and Replace Entries (CAUTION)'}, + options: {'add': 'Add Unique Only', 'update': 'Add and Update existing', 'delete': 'Delete old and Update existing'}, value: feed.duplicateHandle, required: true, }) }}