Skip to content

Commit

Permalink
rework delete handling
Browse files Browse the repository at this point in the history 
Now updates all existing entries, and deletes any not found in the feed.
  • Loading branch information
@engram-design
engram-design committed Jun 21, 2016
1 parent 75ee542 commit 5113318
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 33 deletions.
83 changes: 54 additions & 29 deletions feedme/services/FeedMeService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,46 +8,38 @@ class FeedMeService extends BaseApplicationComponent

public function setupForImport($feed)
{
// Return a collection of only the fields we're mapping
$fields = array();
$return = array(
'fields' => array(),
'processedEntries' => array(),
'existingEntries' => array(),
);

// Start looping through all the mapped fields - checking for nested nodes
foreach ($feed['fieldMapping'] as $itemNode => $destination) {

// Forget about any fields mapped as not to import
if ($destination != 'noimport') {
$fields[$itemNode] = $destination;
$return['fields'][$itemNode] = $destination;
}
}

//
// If our duplication handling is to delete - we delete all entries in this section/entrytype
//

if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete) {
try {

$criteria = craft()->feedMe_entry->setCriteria($feed);
$entries = $criteria->find();

if (!craft()->entries->deleteEntry($entries)) {
FeedMePlugin::log('FeedMeError - Something went wrong while deleting entries.', LogLevel::Error, true);
}
} catch (\Exception $e) {
FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true);
}
$criteria = craft()->feedMe_entry->setCriteria($feed);
$return['existingEntries'] = $criteria->ids();
}

//
// Return variables that need to be used per-node, but only need to be processed once.
//
return array(
'fields' => $fields,
);
return $return;
}

public function importNode($nodes, $feed, $settings)
{
$processedEntries = array();
$hasAnyErrors = false;

$time_start = microtime(true);
Expand All @@ -56,8 +48,12 @@ public function importNode($nodes, $feed, $settings)
foreach ($nodes as $key => $node) {
$result = $this->importSingleNode($node, $feed, $settings);

if (isset($result['entryId'])) {
$processedEntries[] = $result['entryId'];
}

// Report back if even one feed node failed
if (!$result) {
if (!$result['result']) {
$hasAnyErrors = true;
}
}
Expand All @@ -66,7 +62,7 @@ public function importNode($nodes, $feed, $settings)
$execution_time = number_format(($time_end - $time_start), 2);
FeedMePlugin::log($feed->name . ': Processing finished in ' . $execution_time . 's', LogLevel::Info, true);

return !$hasAnyErrors;
return array('result' => !$hasAnyErrors, 'processedEntries' => $processedEntries);
}

public function importSingleNode($node, $feed, $settings)
Expand Down Expand Up @@ -121,7 +117,7 @@ public function importSingleNode($node, $feed, $settings)
} catch (\Exception $e) {
FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true);

return false;
return array('result' => false);
}
}

Expand All @@ -137,16 +133,24 @@ public function importSingleNode($node, $feed, $settings)


// If there's an existing matching entry
if ($existingEntry && $feed['duplicateHandle'] != FeedMe_Duplicate::Delete) {
if ($existingEntry && $feed['duplicateHandle']) {

// If we're deleting
if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete) {

// Fill new EntryModel with match
$entry = $existingEntry;
}

// If we're updating
if ($feed['duplicateHandle'] == FeedMe_Duplicate::Update) {

// Fill new EntryModel with match
$entry = $existingEntry;
}

// If we're adding, make sure not to overwrite existing entry
} else if ($feed['duplicateHandle'] == FeedMe_Duplicate::Add) {
if ($feed['duplicateHandle'] == FeedMe_Duplicate::Add) {
$canSaveEntry = false;
}
} else {
Expand Down Expand Up @@ -179,7 +183,7 @@ public function importSingleNode($node, $feed, $settings)
if (!craft()->entries->saveEntry($entry)) {
FeedMePlugin::log($feed->name . ': ' . json_encode($entry->getErrors()), LogLevel::Error, true);

return false;
return array('result' => false);
} else {

// If we're importing into a specific locale, we need to create this entry if it doesn't already exist
Expand All @@ -193,7 +197,7 @@ public function importSingleNode($node, $feed, $settings)
if (!craft()->entries->saveEntry($entryLocale)) {
FeedMePlugin::log($feed->name . ': ' . json_encode($entryLocale->getErrors()), LogLevel::Error, true);

return false;
return array('result' => false);
} else {

// Successfully saved/added entry
Expand All @@ -203,7 +207,7 @@ public function importSingleNode($node, $feed, $settings)
FeedMePlugin::log($feed->name . ': Entry successfully added: ' . $entryLocale->id, LogLevel::Info, true);
}

return true;
return array('result' => true, 'entryId' => $entryLocale->id);
}
} else {

Expand All @@ -214,20 +218,41 @@ public function importSingleNode($node, $feed, $settings)
FeedMePlugin::log($feed->name . ': Entry successfully added: ' . $entry->id, LogLevel::Info, true);
}

return true;
return array('result' => true, 'entryId' => $entry->id);
}
}
} catch (\Exception $e) {
FeedMePlugin::log($feed->name . ': Entry FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true);

return false;
return array('result' => false, 'entryId' => $entry->id);
}
} else {
if ($existingEntry) {
FeedMePlugin::log($feed->name . ': Entry skipped: ' . $existingEntry->id . '.', LogLevel::Error, true);
}

return true;
return array('result' => true);
}
}

public function deleteLeftoverEntries($settings, $feed, $processedEntries, $result)
{
if ($feed['duplicateHandle'] == FeedMe_Duplicate::Delete && $result['result']) {
$deleteIds = array_diff($settings['existingEntries'], $processedEntries);

$criteria = craft()->feedMe_entry->setCriteria($feed);
$criteria->id = $deleteIds;
$entriesToDelete = $criteria->find();

try {
if (!craft()->entries->deleteEntry($entriesToDelete)) {
FeedMePlugin::log('FeedMeError - Something went wrong while deleting entries.', LogLevel::Error, true);
} else {
FeedMePlugin::log($feed->name . ': The following entries have been deleted: ' . print_r($deleteIds, true) . '.', LogLevel::Error, true);
}
} catch (\Exception $e) {
FeedMePlugin::log($feed->name . ': FeedMeError: ' . $e->getMessage() . '.', LogLevel::Error, true);
}
}
}
}
14 changes: 13 additions & 1 deletion feedme/tasks/FeedMeTask.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class FeedMeTask extends BaseTask
private $_feedSettings;
private $_backup;
private $_chunkedFeedData;
private $_processedEntries = array();

// Public Methods
// =========================================================================
Expand All @@ -35,6 +36,11 @@ public function getTotalSteps()
// Get the data for the mapping screen, based on the URL provided
$this->_feedData = craft()->feedMe_feed->getFeed($this->_feed->feedType, $this->_feed->feedUrl, $this->_feed->primaryElement);

if (!$this->_feedData) {
FeedMePlugin::log($this->_feed->name . ': FeedMeError', LogLevel::Error, true);
return false;
}

// Chunk the feed data into chunks of 100 - optimises mapping process by not calling service each step
$this->_chunkedFeedData = array_chunk($this->_feedData, 100);

Expand All @@ -53,8 +59,14 @@ public function getTotalSteps()
public function runStep($step)
{
$result = craft()->feedMe->importNode($this->_chunkedFeedData[$step], $this->_feed, $this->_feedSettings);
$this->_processedEntries = array_merge($this->_processedEntries, $result['processedEntries']);

// For delete, at the end of our processing, we delete all entries not recorded
if ($step == $this->getTotalSteps()-1) {
craft()->feedMe->deleteLeftoverEntries($this->_feedSettings, $this->_feed, $this->_processedEntries, $result);
}

if (!$result) {
if (!$result['result']) {
return 'Feed Me Failure: Check Feed Me logs.';
} else {
return true;
Expand Down
6 changes: 3 additions & 3 deletions feedme/templates/feeds/_edit.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,11 +117,11 @@
<hr>

{{ forms.selectField({
label: "Import strategy <span class='info'><span style='display:inline-block;'><strong>Add Entries:</strong> Duplicate entries will be skipped, new entries however, will be added. <em>\"I want to keep existing entries untouched but add new ones.\"</em></span><span style='display:inline-block;'><strong>Update Entries:</strong> Select fields to compare duplicate entries with, updating all other fields.<em>\"I want to update existing entries and add new ones.\"</em></span><span style='display:inline-block;'><strong>Delete Entries:</strong> Delete all existing entries in this section, adding only entries from this feed. Be careful.<em>\"I want only the entries from this feed in this section.\"</em></span></span>" | t,
instructions: 'Choose how to handle duplicate records.',
label: "Import strategy <span class='info'><span style='display:inline-block;'><strong>Add Unique Only:</strong> Duplicate entries will be skipped, new entries however, will be added. <em>\"I want to keep existing entries untouched but add new ones.\"</em></span><span style='display:inline-block;'><strong>Add and Update existing:</strong> Select fields to compare duplicate entries with, updating all other fields.<em>\"I want to update existing entries and add new ones.\"</em></span><span style='display:inline-block;'><strong>Delete Entries existing:</strong> Update any existing entries, and delete any entries not updated by this feed. Be careful.<em>\"I want only the entries from this feed in this section.\"</em></span></span>" | t,
instructions: 'Choose what happens when this feed is run again. Click tooltip above for explanation.',
id: 'duplicateHandle',
name: 'duplicateHandle',
options: {'add': 'Add Unique Entries Only', 'update': 'Add and Update Entries', 'delete': 'Delete all and Replace Entries (CAUTION)'},
options: {'add': 'Add Unique Only', 'update': 'Add and Update existing', 'delete': 'Delete old and Update existing'},
value: feed.duplicateHandle,
required: true,
}) }}
Expand Down

0 comments on commit 5113318

Please sign in to comment.