diff --git a/src/libsync/bulkpropagatorjob.cpp b/src/libsync/bulkpropagatorjob.cpp index 08dbadcbca738..9a25201f53c23 100644 --- a/src/libsync/bulkpropagatorjob.cpp +++ b/src/libsync/bulkpropagatorjob.cpp @@ -70,6 +70,7 @@ Q_LOGGING_CATEGORY(lcBulkPropagatorJob, "nextcloud.sync.propagator.bulkupload", BulkPropagatorJob::BulkPropagatorJob(OwncloudPropagator *propagator, const std::deque &items) : PropagatorJob(propagator) , _items(items) + , _currentBatchSize(batchSize) { _filesToUpload.reserve(batchSize); _pendingChecksumFiles.reserve(batchSize); @@ -83,7 +84,7 @@ bool BulkPropagatorJob::scheduleSelfOrChild() _state = Running; - for(auto i = 0; i < batchSize && !_items.empty(); ++i) { + for(auto i = 0; i < _currentBatchSize && !_items.empty(); ++i) { const auto currentItem = _items.front(); _items.pop_front(); _pendingChecksumFiles.insert(currentItem->_file); @@ -107,6 +108,29 @@ bool BulkPropagatorJob::scheduleSelfOrChild() return _items.empty() && _filesToUpload.empty(); } +bool BulkPropagatorJob::handleBatchSize() +{ + // no error, no batch size to change + if (_finalStatus == SyncFileItem::Success || _finalStatus == SyncFileItem::NoStatus) { + qCDebug(lcBulkPropagatorJob) << "No error, no need to change the bulk upload batch size!"; + return true; + } + + // change this value more times? or try only once? + const auto halfBatchSize = batchSize/2; + + // we already tried to upload with half of the batch size + if(_currentBatchSize == halfBatchSize) { + qCDebug(lcBulkPropagatorJob) << "There was another error, stop syncing now!"; + return false; + } + + // try to upload with half of the batch size + _currentBatchSize = halfBatchSize; + qCDebug(lcBulkPropagatorJob) << "There was an error, sync again with bulk upload batch size cut to half!"; + return true; +} + PropagatorJob::JobParallelism BulkPropagatorJob::parallelism() const { return PropagatorJob::JobParallelism::FullParallelism; @@ -254,13 +278,16 @@ void BulkPropagatorJob::checkPropagationIsDone() // just wait for the other job to finish. return; } - - qCInfo(lcBulkPropagatorJob) << "final status" << _finalStatus; - emit finished(_finalStatus); - propagator()->scheduleNextJob(); } else { - scheduleSelfOrChild(); + if (handleBatchSize()) { + scheduleSelfOrChild(); + return; + } } + + qCInfo(lcBulkPropagatorJob) << "final status" << _finalStatus; + emit finished(_finalStatus); + propagator()->scheduleNextJob(); } void BulkPropagatorJob::slotComputeTransmissionChecksum(SyncFileItemPtr item, @@ -371,6 +398,9 @@ void BulkPropagatorJob::slotPutFinishedOneFile(const BulkUploadItem &singleFile, singleFile._item->_status = SyncFileItem::Success; + // upload succeeded, so remove from black list + propagator()->removeFromBulkUploadBlackList(singleFile._item->_file); + // Check the file again post upload. // Two cases must be considered separately: If the upload is finished, // the file is on the server and has a changed ETag. In that case, diff --git a/src/libsync/bulkpropagatorjob.h b/src/libsync/bulkpropagatorjob.h index 51175bfe58ebe..3425b3fe4be18 100644 --- a/src/libsync/bulkpropagatorjob.h +++ b/src/libsync/bulkpropagatorjob.h @@ -155,6 +155,8 @@ private slots: void checkPropagationIsDone(); + bool handleBatchSize(); + std::deque _items; QVector _jobs; /// network jobs that are currently in transit @@ -164,6 +166,7 @@ private slots: std::vector _filesToUpload; SyncFileItem::Status _finalStatus = SyncFileItem::Status::NoStatus; + int _currentBatchSize = 0; }; } diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 488a83a3f0ff5..5c63865497aff 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -404,8 +404,6 @@ std::unique_ptr OwncloudPropagator::createUploadJob(S job->setDeleteExisting(deleteExisting); - removeFromBulkUploadBlackList(item->_file); - return job; } @@ -1261,8 +1259,9 @@ void PropagatorCompositeJob::finalize() { // The propagator will do parallel scheduling and this could be posted // multiple times on the event loop, ignore the duplicate calls. - if (_state == Finished) + if (_state == Finished) { return; + } _state = Finished; emit finished(_hasError == SyncFileItem::NoStatus ? SyncFileItem::Success : _hasError); diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 342ccdd1bb414..6e068163e4327 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -633,8 +633,9 @@ private slots: /** Emit the finished signal and make sure it is only emitted once */ void emitFinished(OCC::SyncFileItem::Status status) { - if (!_finishedEmited) + if (!_finishedEmited) { emit finished(status == SyncFileItem::Success); + } _abortRequested = false; _finishedEmited = true; }