From 3b409a4a273455d8cc8bb5ae84cb3c8bf3afb769 Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Sat, 27 Jul 2019 10:09:24 +0300 Subject: [PATCH 01/32] Improve import speed https://github.com/matchish/laravel-scout-elasticsearch/issues/25 --- src/Jobs/Stages/PullFromSource.php | 31 ++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Jobs/Stages/PullFromSource.php b/src/Jobs/Stages/PullFromSource.php index be34dfea..48b4980d 100644 --- a/src/Jobs/Stages/PullFromSource.php +++ b/src/Jobs/Stages/PullFromSource.php @@ -58,12 +58,31 @@ public static function chunked(Model $searchable): Collection ->orderBy($searchable->getKeyName()); $totalSearchables = $query->count(); if ($totalSearchables) { - $chunkSize = (int) config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); - $totalChunks = (int) ceil($totalSearchables / $chunkSize); - - return collect(range(1, $totalChunks))->map(function ($page) use ($query, $chunkSize) { - $clone = (clone $query)->forPage($page, $chunkSize); - + $chunkSize = (int)config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); + $cloneQuery = clone $query; + $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') + ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); + $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =1 and rownum != 1")->pluck('id'); + $pairs = []; + $lastId = null; + foreach ($ids as $id) { + if ($lastId) { + $pairs[] = [$lastId, $id]; + } else { + $pairs[] = [null, $id]; + } + $lastId = $id; + } + $pairs[] = [$lastId, null]; + return collect($pairs)->map(function ($pair) use ($query, $searchable) { + list($start, $end) = $pair; + $clone = (clone $query) + ->when(!is_null($start), function ($query) use ($start, $searchable) { + return $query->where($searchable->getKeyName(), '>', $start); + }) + ->when(!is_null($end), function ($query) use ($end, $searchable) { + return $query->where($searchable->getKeyName(), '<=', $end); + }); return new static($clone); }); } else { From c1591afe4abba3c59fa06b94dbfa75cec4c11d3b Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:46:06 +0300 Subject: [PATCH 02/32] Add mysql container --- docker-compose.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a469effc..253132d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,19 @@ services: XDEBUG_CONFIG: ${XDEBUG_CONFIG} PHP_IDE_CONFIG: ${PHP_IDE_CONFIG} ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST} + DB_HOST: ${DB_HOST} + DB_DATABASE: ${DB_DATABASE} + DB_USERNAME: ${DB_USERNAME} + DB_PASSWORD: ${DB_PASSWORD} + networks: + - default + db: + image: mysql:5.7.22 + environment: + MYSQL_DATABASE: ${DB_DATABASE} + MYSQL_USER: ${DB_USERNAME} + MYSQL_PASSWORD: ${DB_PASSWORD} + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD} networks: - default elasticsearch: From facb84374cd9d4501d9ed955fb4fbee1c41e1f13 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:46:48 +0300 Subject: [PATCH 03/32] Using mysql instead sqlite in tests --- tests/TestCase.php | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/TestCase.php b/tests/TestCase.php index 0e41c408..c627a296 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -17,17 +17,17 @@ public function setUp(): void { parent::setUp(); - $this->app->setBasePath(__DIR__.'/laravel'); + $this->app->setBasePath(__DIR__ . '/laravel'); $this->withFactories(database_path('factories')); - Artisan::call('migrate:fresh', ['--database' => 'testbench']); + Artisan::call('migrate:fresh', ['--database' => 'mysql']); } /** * Define environment setup. * - * @param \Illuminate\Foundation\Application $app + * @param \Illuminate\Foundation\Application $app * @return void */ protected function getEnvironmentSetUp($app) @@ -36,13 +36,8 @@ protected function getEnvironmentSetUp($app) $app['config']->set('scout.chunk.searchable', 3); $app['config']->set('scout.queue', false); // Setup default database to use sqlite :memory: - $app['config']->set('database.default', 'testbench'); - $app['config']->set('database.connections.testbench', [ - 'driver' => 'sqlite', - 'database' => ':memory:', - 'prefix' => '', - ]); - $app['config']->set('elasticsearch', require(__DIR__.'/../config/elasticsearch.php')); + $app['config']->set('database.default', 'mysql'); + $app['config']->set('elasticsearch', require(__DIR__ . '/../config/elasticsearch.php')); $app['config']->set('elasticsearch.indices.mappings.products', [ '_doc' => [ 'properties' => [ @@ -54,7 +49,7 @@ protected function getEnvironmentSetUp($app) ], ], ], - ]); + ]); } protected function getPackageProviders($app) From 8bee984be3be5d5f16a43af654d91ec268d0dd4d Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:47:16 +0300 Subject: [PATCH 04/32] Add mysql service in travis config --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index bc90eda5..14b11128 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: php env: - ELASTICSEARCH_HOST=localhost:9200 + - ELASTICSEARCH_HOST=localhost:9200 + - DB_DATABASE=scout_database + - DB_USERNAME=travis + - DB_PASSWORD="" php: - 7.1 @@ -18,6 +22,7 @@ matrix: - php: nightly before_script: + - mysql -e 'create database $DB_DATABASE;' - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.2.deb && sudo dpkg -i --force-confnew elasticsearch-6.6.2.deb && sudo service elasticsearch restart - sleep 20 - curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=50s" From 986bf4c66c0b1e8630dac088fdf2d3239bf9a1e9 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:48:18 +0300 Subject: [PATCH 05/32] Fix chunking query --- src/Jobs/Stages/PullFromSource.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Jobs/Stages/PullFromSource.php b/src/Jobs/Stages/PullFromSource.php index 48b4980d..b90b8ec0 100644 --- a/src/Jobs/Stages/PullFromSource.php +++ b/src/Jobs/Stages/PullFromSource.php @@ -62,7 +62,7 @@ public static function chunked(Model $searchable): Collection $cloneQuery = clone $query; $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); - $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =1 and rownum != 1")->pluck('id'); + $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck('id'); $pairs = []; $lastId = null; foreach ($ids as $id) { From 831534524fab261a594671a19ade328de21c6a71 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:55:21 +0300 Subject: [PATCH 06/32] Add mysql service to scrutinizer config --- .scrutinizer.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 726f7d38..9817fdae 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -16,8 +16,12 @@ build: environment: variables: ELASTICSEARCH_HOST: 'localhost:9200' + DB_HOST: '127.0.0.1' + DB_USERNAME: 'root' + DB_PASSWORD: '' services: elasticsearch: 6.6.2 + mysql: 5.7 tests: override: - php-scrutinizer-run @@ -25,4 +29,4 @@ build: command: 'vendor/bin/phpunit --coverage-clover=coverage.clover' coverage: file: 'coverage.clover' - format: 'clover' \ No newline at end of file + format: 'clover' From ced38935920c1118abe0f31df3f2f1093ca03c37 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 09:58:17 +0300 Subject: [PATCH 07/32] Fix travis config --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14b11128..35b48686 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,7 @@ language: php env: - - ELASTICSEARCH_HOST=localhost:9200 - - ELASTICSEARCH_HOST=localhost:9200 - - DB_DATABASE=scout_database - - DB_USERNAME=travis - - DB_PASSWORD="" + - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" php: - 7.1 From 7299405a54f6961ad8de317fce7701ba252afe5d Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 11:57:32 +0300 Subject: [PATCH 08/32] Fix scrutinizer config --- .scrutinizer.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 9817fdae..3e163271 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -11,14 +11,13 @@ build: environment: php: version: '7.2' + variables: + ELASTICSEARCH_HOST: 'localhost:9200' + DB_HOST: '127.0.0.1' + DB_USERNAME: 'root' + DB_PASSWORD: '' nodes: analysis: - environment: - variables: - ELASTICSEARCH_HOST: 'localhost:9200' - DB_HOST: '127.0.0.1' - DB_USERNAME: 'root' - DB_PASSWORD: '' services: elasticsearch: 6.6.2 mysql: 5.7 From 46547dc010cbb97a176739a3291ed442c7d4a9b1 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sun, 28 Jul 2019 12:59:34 +0300 Subject: [PATCH 09/32] Create database in scrutinizer --- .scrutinizer.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 3e163271..4bc0ba42 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -16,6 +16,10 @@ build: DB_HOST: '127.0.0.1' DB_USERNAME: 'root' DB_PASSWORD: '' + DB_DATABASE: 'my_database' + project_setup: + before: + - mysql -e "CREATE DATABASE my_database" nodes: analysis: services: From 945f3c8b846193c2f93fa4d0149d4614d9611137 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Mon, 5 Aug 2019 05:58:09 +0300 Subject: [PATCH 10/32] Decouple import from searchable trait --- src/Console/Commands/ChunkScope.php | 57 ++++++++++ src/Console/Commands/DefaultImportSource.php | 102 ++++++++++++++++++ .../Commands/DefaultImportSourceFactory.php | 15 +++ src/Console/Commands/ImportCommand.php | 9 +- src/Console/Commands/ImportSource.php | 18 ++++ src/Console/Commands/ImportSourceFactory.php | 10 ++ .../EloquentHitsIteratorAggregate.php | 9 +- src/ElasticSearch/Index.php | 10 +- src/Jobs/Import.php | 13 +-- src/Jobs/ImportStages.php | 16 +-- src/Jobs/Stages/CleanUp.php | 18 ++-- src/Jobs/Stages/CreateWriteIndex.php | 18 ++-- src/Jobs/Stages/PullFromSource.php | 64 +++-------- .../Stages/SwitchToNewAndRemoveOldIndex.php | 21 ++-- src/ScoutElasticSearchServiceProvider.php | 3 + tests/Integration/Jobs/ImportTest.php | 3 +- tests/Integration/Jobs/Stages/CleanUpTest.php | 3 +- .../Jobs/Stages/CreateWriteIndexTest.php | 3 +- .../Jobs/Stages/PullFromSourceTest.php | 15 +-- .../SwitchToNewAndRemoveOldIndexTest.php | 3 +- tests/IntegrationTestCase.php | 6 -- tests/Unit/ElasticSearch/IndexTest.php | 9 +- tests/Unit/Jobs/ImportStagesTest.php | 5 +- 23 files changed, 293 insertions(+), 137 deletions(-) create mode 100644 src/Console/Commands/ChunkScope.php create mode 100644 src/Console/Commands/DefaultImportSource.php create mode 100644 src/Console/Commands/DefaultImportSourceFactory.php create mode 100644 src/Console/Commands/ImportSource.php create mode 100644 src/Console/Commands/ImportSourceFactory.php diff --git a/src/Console/Commands/ChunkScope.php b/src/Console/Commands/ChunkScope.php new file mode 100644 index 00000000..67147a1f --- /dev/null +++ b/src/Console/Commands/ChunkScope.php @@ -0,0 +1,57 @@ +start = $start; + $this->end = $end; + } + + /** + * Apply the scope to a given Eloquent query builder. + * + * @param \Illuminate\Database\Eloquent\Builder $builder + * @param \Illuminate\Database\Eloquent\Model $model + * @return void + */ + public function apply(Builder $builder, Model $model) + { + $start = $this->start; + $end = $this->end; + $builder + ->when(!is_null($start), function ($query) use ($start, $model) { + return $query->where($model->getKeyName(), '>', $start); + }) + ->when(!is_null($end), function ($query) use ($end, $model) { + return $query->where($model->getKeyName(), '<=', $end); + }); + } + + public function key() + { + return static::class; + } + +} diff --git a/src/Console/Commands/DefaultImportSource.php b/src/Console/Commands/DefaultImportSource.php new file mode 100644 index 00000000..b79a9d96 --- /dev/null +++ b/src/Console/Commands/DefaultImportSource.php @@ -0,0 +1,102 @@ +className = $className; + $this->scopes = $scopes; + } + + public function syncWithSearchUsingQueue() + { + return $this->model()->syncWithSearchUsingQueue(); + } + + + public function syncWithSearchUsing() + { + return $this->model()->syncWithSearchUsing(); + } + + public function searchableAs(): string + { + return $this->model()->searchableAs(); + } + + public function chunked() + { + $query = $this->newQuery(); + $searchable = $this->model(); + $totalSearchables = $query->count(); + if ($totalSearchables) { + $chunkSize = (int)config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); + $cloneQuery = clone $query; + $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') + ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); + $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck('id'); + $pairs = []; + $lastId = null; + foreach ($ids as $id) { + if ($lastId) { + $pairs[] = [$lastId, $id]; + } else { + $pairs[] = [null, $id]; + } + $lastId = $id; + } + $pairs[] = [$lastId, null]; + return collect($pairs)->map(function ($pair) { + list($start, $end) = $pair; + $chunkScope = new ChunkScope($start, $end); + return new static($this->className, [$chunkScope]); + }); + } else { + return collect(); + } + + } + + /** + * @return mixed + */ + private function model() + { + return new $this->className; + } + + private function newQuery() + { + $query = $this->model()->newQuery(); + $softDelete = config('scout.soft_delete', false); + $query + ->when($softDelete, function ($query) { + return $query->withTrashed(); + }) + ->orderBy($this->model()->getKeyName()); + $scopes = $this->scopes; + return collect($scopes)->reduce(function ($instance, $scope) { + $instance->withGlobalScope($scope->key(), $scope); + return $instance; + }, $query); + } + + public function get() + { + return $this->newQuery()->get(); + } +} diff --git a/src/Console/Commands/DefaultImportSourceFactory.php b/src/Console/Commands/DefaultImportSourceFactory.php new file mode 100644 index 00000000..5e5b6585 --- /dev/null +++ b/src/Console/Commands/DefaultImportSourceFactory.php @@ -0,0 +1,15 @@ +chain([$job]); @@ -54,8 +56,9 @@ private function import($searchable) $startMessage = trans('scout::import.start', ['searchable' => "$searchable"]); $this->line($startMessage); - dispatch($job)->allOnQueue((new $searchable)->syncWithSearchUsingQueue()) - ->allOnConnection(config((new $searchable)->syncWithSearchUsing())); + /** @var ImportSource $source */ + dispatch($job)->allOnQueue($source->syncWithSearchUsingQueue()) + ->allOnConnection($source->syncWithSearchUsing()); $doneMessage = trans(config('scout.queue') ? 'scout::import.done.queue' : 'scout::import.done', [ 'searchable' => $searchable, diff --git a/src/Console/Commands/ImportSource.php b/src/Console/Commands/ImportSource.php new file mode 100644 index 00000000..f5a3c7e3 --- /dev/null +++ b/src/Console/Commands/ImportSource.php @@ -0,0 +1,18 @@ +searchableAs().'_'.time(); - $settingsConfigKey = "elasticsearch.indices.settings.{$searchable->searchableAs()}"; - $mappingsConfigKey = "elasticsearch.indices.mappings.{$searchable->searchableAs()}"; + $name = $source->searchableAs().'_'.time(); + $settingsConfigKey = "elasticsearch.indices.settings.{$source->searchableAs()}"; + $mappingsConfigKey = "elasticsearch.indices.mappings.{$source->searchableAs()}"; $defaultSettings = [ 'number_of_shards' => 1, 'number_of_replicas' => 0, diff --git a/src/Jobs/Import.php b/src/Jobs/Import.php index 139c09d6..c8031288 100644 --- a/src/Jobs/Import.php +++ b/src/Jobs/Import.php @@ -5,6 +5,7 @@ use Elasticsearch\Client; use Illuminate\Bus\Queueable; use Illuminate\Support\Collection; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; use Matchish\ScoutElasticSearch\ProgressReportable; /** @@ -16,16 +17,16 @@ final class Import use ProgressReportable; /** - * @var string + * @var ImportSource */ - private $searchable; + private $source; /** - * @param string $searchable + * @param ImportSource $source */ - public function __construct(string $searchable) + public function __construct(ImportSource $source) { - $this->searchable = $searchable; + $this->source = $source; } /** @@ -45,6 +46,6 @@ public function handle(Client $elasticsearch): void private function stages(): Collection { - return ImportStages::fromSearchable(new $this->searchable); + return ImportStages::fromSource($this->source); } } diff --git a/src/Jobs/ImportStages.php b/src/Jobs/ImportStages.php index b94d675c..cdfd9da8 100644 --- a/src/Jobs/ImportStages.php +++ b/src/Jobs/ImportStages.php @@ -3,7 +3,7 @@ namespace Matchish\ScoutElasticSearch\Jobs; use Illuminate\Support\Collection; -use Illuminate\Database\Eloquent\Model; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex; @@ -14,19 +14,19 @@ class ImportStages extends Collection { /** - * @param Model $searchable + * @param ImportSource $source * @return Collection */ - public static function fromSearchable(Model $searchable) + public static function fromSource(ImportSource $source) { - $index = Index::fromSearchable($searchable); + $index = Index::fromSource($source); return (new static([ - new CleanUp($searchable), - new CreateWriteIndex($searchable, $index), - PullFromSource::chunked($searchable), + new CleanUp($source), + new CreateWriteIndex($source, $index), + PullFromSource::chunked($source), new RefreshIndex($index), - new SwitchToNewAndRemoveOldIndex($searchable, $index), + new SwitchToNewAndRemoveOldIndex($source, $index), ]))->flatten()->filter(); } } diff --git a/src/Jobs/Stages/CleanUp.php b/src/Jobs/Stages/CleanUp.php index c3d351ed..290588b7 100644 --- a/src/Jobs/Stages/CleanUp.php +++ b/src/Jobs/Stages/CleanUp.php @@ -3,9 +3,8 @@ namespace Matchish\ScoutElasticSearch\Jobs\Stages; use Elasticsearch\Client; -use Laravel\Scout\Searchable; -use Illuminate\Database\Eloquent\Model; use Elasticsearch\Common\Exceptions\Missing404Exception; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get as GetAliasParams; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Delete as DeleteIndexParams; @@ -15,23 +14,22 @@ final class CleanUp { /** - * @var Model + * @var ImportSource */ - private $searchable; + private $source; /** - * @param Model $searchable + * @param ImportSource $source */ - public function __construct(Model $searchable) + public function __construct(ImportSource $source) { - $this->searchable = $searchable; + $this->source = $source; } public function handle(Client $elasticsearch): void { - /** @var Searchable $searchable */ - $searchable = $this->searchable; - $params = GetAliasParams::anyIndex($searchable->searchableAs()); + $source = $this->source; + $params = GetAliasParams::anyIndex($source->searchableAs()); try { $response = $elasticsearch->indices()->getAliases($params->toArray()); } catch (Missing404Exception $e) { diff --git a/src/Jobs/Stages/CreateWriteIndex.php b/src/Jobs/Stages/CreateWriteIndex.php index 6a7799e3..4462ea92 100644 --- a/src/Jobs/Stages/CreateWriteIndex.php +++ b/src/Jobs/Stages/CreateWriteIndex.php @@ -3,8 +3,7 @@ namespace Matchish\ScoutElasticSearch\Jobs\Stages; use Elasticsearch\Client; -use Laravel\Scout\Searchable; -use Illuminate\Database\Eloquent\Model; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\ElasticSearch\WriteAlias; use Matchish\ScoutElasticSearch\ElasticSearch\DefaultAlias; @@ -16,29 +15,28 @@ final class CreateWriteIndex { /** - * @var Model + * @var ImportSource */ - private $searchable; + private $source; /** * @var Index */ private $index; /** - * @param Model $searchable + * @param ImportSource $source * @param Index $index */ - public function __construct(Model $searchable, Index $index) + public function __construct(ImportSource $source, Index $index) { - $this->searchable = $searchable; + $this->source = $source; $this->index = $index; } public function handle(Client $elasticsearch): void { - /** @var Searchable $searchable */ - $searchable = $this->searchable; - $this->index->addAlias(new WriteAlias(new DefaultAlias($searchable->searchableAs()))); + $source = $this->source; + $this->index->addAlias(new WriteAlias(new DefaultAlias($source->searchableAs()))); $params = new Create( $this->index->name(), diff --git a/src/Jobs/Stages/PullFromSource.php b/src/Jobs/Stages/PullFromSource.php index b90b8ec0..317e6c90 100644 --- a/src/Jobs/Stages/PullFromSource.php +++ b/src/Jobs/Stages/PullFromSource.php @@ -2,34 +2,30 @@ namespace Matchish\ScoutElasticSearch\Jobs\Stages; -use Laravel\Scout\Searchable; use Illuminate\Support\Collection; -use Illuminate\Database\Eloquent\Model; -use Illuminate\Database\Eloquent\Builder; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; /** * @internal */ final class PullFromSource { - const DEFAULT_CHUNK_SIZE = 500; - /** - * @var Builder + * @var ImportSource */ - private $query; + private $source; /** - * @param Builder $query + * @param ImportSource $source */ - public function __construct(Builder $query) + public function __construct(ImportSource $source) { - $this->query = $query; + $this->source = $source; } public function handle(): void { - $results = $this->query->get(); + $results = $this->source->get(); $results->filter->shouldBeSearchable()->searchable(); } @@ -44,49 +40,13 @@ public function title(): string } /** - * @param Model $searchable + * @param ImportSource $source * @return Collection */ - public static function chunked(Model $searchable): Collection + public static function chunked(ImportSource $source): Collection { - /** @var Searchable $searchable */ - $softDelete = config('scout.soft_delete', false); - $query = $searchable->newQuery() - ->when($softDelete, function ($query) { - return $query->withTrashed(); - }) - ->orderBy($searchable->getKeyName()); - $totalSearchables = $query->count(); - if ($totalSearchables) { - $chunkSize = (int)config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); - $cloneQuery = clone $query; - $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') - ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); - $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck('id'); - $pairs = []; - $lastId = null; - foreach ($ids as $id) { - if ($lastId) { - $pairs[] = [$lastId, $id]; - } else { - $pairs[] = [null, $id]; - } - $lastId = $id; - } - $pairs[] = [$lastId, null]; - return collect($pairs)->map(function ($pair) use ($query, $searchable) { - list($start, $end) = $pair; - $clone = (clone $query) - ->when(!is_null($start), function ($query) use ($start, $searchable) { - return $query->where($searchable->getKeyName(), '>', $start); - }) - ->when(!is_null($end), function ($query) use ($end, $searchable) { - return $query->where($searchable->getKeyName(), '<=', $end); - }); - return new static($clone); - }); - } else { - return collect(); - } + return $source->chunked()->map(function ($chunk) { + return new static($chunk); + }); } } diff --git a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php index 025c7242..ad47f893 100644 --- a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php +++ b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php @@ -3,8 +3,7 @@ namespace Matchish\ScoutElasticSearch\Jobs\Stages; use Elasticsearch\Client; -use Laravel\Scout\Searchable; -use Illuminate\Database\Eloquent\Model; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Update; @@ -15,28 +14,28 @@ final class SwitchToNewAndRemoveOldIndex { /** - * @var Model + * @var ImportSource */ - private $searchable; + private $source; /** * @var Index */ private $index; /** - * @param Model $searchable + * @param ImportSource $source + * @param Index $index */ - public function __construct(Model $searchable, Index $index) + public function __construct(ImportSource $source, Index $index) { - $this->searchable = $searchable; + $this->source = $source; $this->index = $index; } public function handle(Client $elasticsearch): void { - /** @var Searchable $searchable */ - $searchable = $this->searchable; - $params = Get::anyIndex($searchable->searchableAs()); + $source = $this->source; + $params = Get::anyIndex($source->searchableAs()); $response = $elasticsearch->indices()->getAliases($params->toArray()); $params = new Update(); @@ -44,7 +43,7 @@ public function handle(Client $elasticsearch): void if ($indexName != $this->index->name()) { $params->removeIndex($indexName); } else { - $params->add((string) $indexName, $searchable->searchableAs()); + $params->add((string) $indexName, $source->searchableAs()); } } $elasticsearch->indices()->updateAliases($params->toArray()); diff --git a/src/ScoutElasticSearchServiceProvider.php b/src/ScoutElasticSearchServiceProvider.php index ff1e3870..62384a68 100644 --- a/src/ScoutElasticSearchServiceProvider.php +++ b/src/ScoutElasticSearchServiceProvider.php @@ -8,6 +8,8 @@ use Laravel\Scout\EngineManager; use Illuminate\Support\ServiceProvider; use Laravel\Scout\ScoutServiceProvider; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; +use Matchish\ScoutElasticSearch\Console\Commands\ImportSourceFactory; use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; use Matchish\ScoutElasticSearch\Console\Commands\FlushCommand; use Matchish\ScoutElasticSearch\Console\Commands\ImportCommand; @@ -34,6 +36,7 @@ public function boot(): void public function register(): void { $this->app->register(ScoutServiceProvider::class); + $this->app->bind(ImportSourceFactory::class, DefaultImportSourceFactory::class); $this->registerCommands(); } diff --git a/tests/Integration/Jobs/ImportTest.php b/tests/Integration/Jobs/ImportTest.php index 60b55068..42878c54 100644 --- a/tests/Integration/Jobs/ImportTest.php +++ b/tests/Integration/Jobs/ImportTest.php @@ -3,6 +3,7 @@ namespace Tests\Integration\Jobs; use App\Product; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use Tests\IntegrationTestCase; use Illuminate\Console\OutputStyle; use Matchish\ScoutElasticSearch\Jobs\Import; @@ -19,7 +20,7 @@ public function test_progress_report() factory(Product::class, $productsAmount)->create(); Product::setEventDispatcher($dispatcher); - $job = new Import(Product::class); + $job = new Import(DefaultImportSourceFactory::from(Product::class)); $output = new DummyOutput(); $outputStyle = new OutputStyle(new ArrayInput([]), $output); $progressBar = $outputStyle->createProgressBar(); diff --git a/tests/Integration/Jobs/Stages/CleanUpTest.php b/tests/Integration/Jobs/Stages/CleanUpTest.php index ca2c6442..7726d12d 100644 --- a/tests/Integration/Jobs/Stages/CleanUpTest.php +++ b/tests/Integration/Jobs/Stages/CleanUpTest.php @@ -2,6 +2,7 @@ namespace Tests\Integration\Jobs\Stages; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use stdClass; use App\Product; use Tests\IntegrationTestCase; @@ -24,7 +25,7 @@ public function test_remove_write_index() 'body' => ['aliases' => ['products' => ['is_write_index' => false]]], ]); - $stage = new CleanUp(new Product()); + $stage = new CleanUp(DefaultImportSourceFactory::from(Product::class)); $stage->handle($this->elasticsearch); $writeIndexExist = $this->elasticsearch->indices()->exists(['index' => 'products_new']); $readIndexExist = $this->elasticsearch->indices()->exists(['index' => 'products_old']); diff --git a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php index 0877a117..0cc44410 100644 --- a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php +++ b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php @@ -6,6 +6,7 @@ use App\Product; use Elasticsearch\Client; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use Tests\IntegrationTestCase; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex; @@ -15,7 +16,7 @@ final class CreateWriteIndexTest extends IntegrationTestCase public function test_create_write_index(): void { $elasticsearch = $this->app->make(Client::class); - $stage = new CreateWriteIndex(new Product(), Index::fromSearchable(new Product())); + $stage = new CreateWriteIndex(DefaultImportSourceFactory::from(Product::class), Index::fromSource(DefaultImportSourceFactory::from(Product::class))); $stage->handle($elasticsearch); $response = $elasticsearch->indices()->getAliases(['index' => '*', 'name' => 'products']); $this->assertTrue($this->containsWriteIndex($response, 'products')); diff --git a/tests/Integration/Jobs/Stages/PullFromSourceTest.php b/tests/Integration/Jobs/Stages/PullFromSourceTest.php index f54851d6..684fca29 100644 --- a/tests/Integration/Jobs/Stages/PullFromSourceTest.php +++ b/tests/Integration/Jobs/Stages/PullFromSourceTest.php @@ -4,6 +4,7 @@ namespace Tests\Integration\Jobs\Stages; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use stdClass; use App\Product; use Tests\IntegrationTestCase; @@ -25,7 +26,7 @@ public function test_put_all_entites_to_index(): void 'index' => 'products_index', 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stage = new PullFromSource(Product::query()); + $stage = new PullFromSource(DefaultImportSourceFactory::from(Product::class)); $stage->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', @@ -48,7 +49,7 @@ public function test_dont_put_entities_if_no_entities_in_collection(): void 'index' => 'products_index', 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stage = new PullFromSource(Product::query()); + $stage = new PullFromSource(DefaultImportSourceFactory::from(Product::class)); $stage->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', @@ -80,7 +81,7 @@ public function test_put_all_to_index_if_amount_of_entities_more_than_chunk_size 'index' => 'products_index', 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stage = new PullFromSource(Product::query()); + $stage = new PullFromSource(DefaultImportSourceFactory::from(Product::class)); $stage->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', @@ -113,7 +114,7 @@ public function test_pull_soft_delete_meta_data() 'index' => 'products_index', 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stage = new PullFromSource(Product::query()); + $stage = new PullFromSource(DefaultImportSourceFactory::from(Product::class)); $stage->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', @@ -148,7 +149,7 @@ public function test_pull_soft_deleted() 'index' => 'products_index', 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stages = PullFromSource::chunked(new Product()); + $stages = PullFromSource::chunked(DefaultImportSourceFactory::from(Product::class)); $stages->first()->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', @@ -167,7 +168,7 @@ public function test_pull_soft_deleted() public function test_no_searchables_no_chunks() { - $stages = PullFromSource::chunked(new Product()); + $stages = PullFromSource::chunked(DefaultImportSourceFactory::from(Product::class)); $this->assertEquals(0, $stages->count()); } @@ -183,7 +184,7 @@ public function test_chunked_pull_only_one_page() Product::setEventDispatcher($dispatcher); - $chunks = PullFromSource::chunked(new Product()); + $chunks = PullFromSource::chunked(DefaultImportSourceFactory::from(Product::class)); $chunks->first()->handle(); $this->elasticsearch->indices()->refresh([ 'index' => 'products', diff --git a/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php b/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php index f608f2ec..36037ff8 100644 --- a/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php +++ b/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php @@ -4,6 +4,7 @@ namespace Tests\Integration\Jobs\Stages; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use stdClass; use App\Product; use Tests\IntegrationTestCase; @@ -23,7 +24,7 @@ public function test_switch_to_new_and_remove_old_index(): void 'body' => ['aliases' => ['products' => new stdClass()]], ]); - $stage = new SwitchToNewAndRemoveOldIndex(new Product(), new Index('products_new')); + $stage = new SwitchToNewAndRemoveOldIndex(DefaultImportSourceFactory::from(Product::class), new Index('products_new')); $stage->handle($this->elasticsearch); $newIndexExist = $this->elasticsearch->indices()->exists(['index' => 'products_new']); diff --git a/tests/IntegrationTestCase.php b/tests/IntegrationTestCase.php index 65793ccd..b25fb8e8 100644 --- a/tests/IntegrationTestCase.php +++ b/tests/IntegrationTestCase.php @@ -1,10 +1,4 @@ assertEquals($index->name(), 'products_1525376494'); } } diff --git a/tests/Unit/Jobs/ImportStagesTest.php b/tests/Unit/Jobs/ImportStagesTest.php index 55d8fe78..0101abe1 100644 --- a/tests/Unit/Jobs/ImportStagesTest.php +++ b/tests/Unit/Jobs/ImportStagesTest.php @@ -3,6 +3,7 @@ namespace Tests\Unit\Jobs; use App\Product; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use Tests\TestCase; use Matchish\ScoutElasticSearch\Jobs\ImportStages; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; @@ -15,7 +16,7 @@ class ImportStagesTest extends TestCase { public function test_no_pull_stages_if_no_searchables() { - $stages = ImportStages::fromSearchable(new Product()); + $stages = ImportStages::fromSource(DefaultImportSourceFactory::from(Product::class)); $this->assertEquals(4, $stages->count()); $this->assertInstanceOf(CleanUp::class, $stages->get(0)); $this->assertInstanceOf(CreateWriteIndex::class, $stages->get(1)); @@ -26,7 +27,7 @@ public function test_no_pull_stages_if_no_searchables() public function test_stages() { factory(Product::class, 10)->create(); - $stages = ImportStages::fromSearchable(new Product()); + $stages = ImportStages::fromSource(DefaultImportSourceFactory::from(Product::class)); $this->assertEquals(8, $stages->count()); $this->assertInstanceOf(CleanUp::class, $stages->get(0)); $this->assertInstanceOf(CreateWriteIndex::class, $stages->get(1)); From 7204d010a24b1df409df4d3c0d2ec866c057459a Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Tue, 6 Aug 2019 06:12:35 +0300 Subject: [PATCH 11/32] Add mysql service to travis --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 35b48686..484bf141 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: php -env: +services: + - mysql + + env: - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" php: From 780c9fafb3804a730e19d16ee1af8c78f12dc69f Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Tue, 6 Aug 2019 06:27:11 +0300 Subject: [PATCH 12/32] Fix travis config formatting --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 484bf141..b492288d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ language: php services: - mysql - env: +env: - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" php: From c840fc84d753f003347d34dace13ba220c728238 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Tue, 6 Aug 2019 08:17:37 +0300 Subject: [PATCH 13/32] Fix travis config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b492288d..91dee885 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: php services: - mysql - + env: - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" @@ -21,9 +21,9 @@ matrix: - php: nightly before_script: - - mysql -e 'create database $DB_DATABASE;' + - mysql -e "create database $DB_DATABASE;" - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.2.deb && sudo dpkg -i --force-confnew elasticsearch-6.6.2.deb && sudo service elasticsearch restart - - sleep 20 + - sleep 30 - curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=50s" - travis_retry composer self-update - travis_retry composer update --no-interaction --no-suggest --prefer-source From 0746c645d97b00a44057e5c79c43728e0ae04775 Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Tue, 6 Aug 2019 12:55:27 +0000 Subject: [PATCH 14/32] Apply fixes from StyleCI --- src/Console/Commands/ChunkScope.php | 8 +++----- src/Console/Commands/DefaultImportSource.php | 12 ++++++------ src/Console/Commands/DefaultImportSourceFactory.php | 4 ---- src/Console/Commands/ImportCommand.php | 2 +- src/Console/Commands/ImportSource.php | 2 -- src/Console/Commands/ImportSourceFactory.php | 2 -- src/ElasticSearch/EloquentHitsIteratorAggregate.php | 2 +- src/Engines/ElasticSearchEngine.php | 1 - src/Jobs/Import.php | 2 +- src/Jobs/ImportStages.php | 2 +- src/Jobs/Stages/CreateWriteIndex.php | 2 +- src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php | 2 +- src/ScoutElasticSearchServiceProvider.php | 4 ++-- tests/Integration/Jobs/ImportTest.php | 2 +- tests/Integration/Jobs/Stages/CleanUpTest.php | 2 +- .../Integration/Jobs/Stages/CreateWriteIndexTest.php | 2 +- tests/Integration/Jobs/Stages/PullFromSourceTest.php | 2 +- .../Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php | 2 +- tests/TestCase.php | 4 ++-- tests/Unit/ElasticSearch/IndexTest.php | 2 +- tests/Unit/Jobs/ImportStagesTest.php | 2 +- 21 files changed, 26 insertions(+), 37 deletions(-) diff --git a/src/Console/Commands/ChunkScope.php b/src/Console/Commands/ChunkScope.php index 67147a1f..835f4b5d 100644 --- a/src/Console/Commands/ChunkScope.php +++ b/src/Console/Commands/ChunkScope.php @@ -1,11 +1,10 @@ start; $end = $this->end; $builder - ->when(!is_null($start), function ($query) use ($start, $model) { + ->when(! is_null($start), function ($query) use ($start, $model) { return $query->where($model->getKeyName(), '>', $start); }) - ->when(!is_null($end), function ($query) use ($end, $model) { + ->when(! is_null($end), function ($query) use ($end, $model) { return $query->where($model->getKeyName(), '<=', $end); }); } @@ -53,5 +52,4 @@ public function key() { return static::class; } - } diff --git a/src/Console/Commands/DefaultImportSource.php b/src/Console/Commands/DefaultImportSource.php index b79a9d96..ecdf41e2 100644 --- a/src/Console/Commands/DefaultImportSource.php +++ b/src/Console/Commands/DefaultImportSource.php @@ -1,9 +1,7 @@ model()->syncWithSearchUsingQueue(); } - public function syncWithSearchUsing() { return $this->model()->syncWithSearchUsing(); @@ -44,7 +41,7 @@ public function chunked() $searchable = $this->model(); $totalSearchables = $query->count(); if ($totalSearchables) { - $chunkSize = (int)config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); + $chunkSize = (int) config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); $cloneQuery = clone $query; $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); @@ -60,15 +57,16 @@ public function chunked() $lastId = $id; } $pairs[] = [$lastId, null]; + return collect($pairs)->map(function ($pair) { - list($start, $end) = $pair; + [$start, $end] = $pair; $chunkScope = new ChunkScope($start, $end); + return new static($this->className, [$chunkScope]); }); } else { return collect(); } - } /** @@ -89,8 +87,10 @@ private function newQuery() }) ->orderBy($this->model()->getKeyName()); $scopes = $this->scopes; + return collect($scopes)->reduce(function ($instance, $scope) { $instance->withGlobalScope($scope->key(), $scope); + return $instance; }, $query); } diff --git a/src/Console/Commands/DefaultImportSourceFactory.php b/src/Console/Commands/DefaultImportSourceFactory.php index 5e5b6585..6e47df89 100644 --- a/src/Console/Commands/DefaultImportSourceFactory.php +++ b/src/Console/Commands/DefaultImportSourceFactory.php @@ -1,13 +1,9 @@ "$searchable"]); $this->line($startMessage); - /** @var ImportSource $source */ + /* @var ImportSource $source */ dispatch($job)->allOnQueue($source->syncWithSearchUsingQueue()) ->allOnConnection($source->syncWithSearchUsing()); diff --git a/src/Console/Commands/ImportSource.php b/src/Console/Commands/ImportSource.php index f5a3c7e3..c04052eb 100644 --- a/src/Console/Commands/ImportSource.php +++ b/src/Console/Commands/ImportSource.php @@ -1,9 +1,7 @@ app->setBasePath(__DIR__ . '/laravel'); + $this->app->setBasePath(__DIR__.'/laravel'); $this->withFactories(database_path('factories')); @@ -37,7 +37,7 @@ protected function getEnvironmentSetUp($app) $app['config']->set('scout.queue', false); // Setup default database to use sqlite :memory: $app['config']->set('database.default', 'mysql'); - $app['config']->set('elasticsearch', require(__DIR__ . '/../config/elasticsearch.php')); + $app['config']->set('elasticsearch', require(__DIR__.'/../config/elasticsearch.php')); $app['config']->set('elasticsearch.indices.mappings.products', [ '_doc' => [ 'properties' => [ diff --git a/tests/Unit/ElasticSearch/IndexTest.php b/tests/Unit/ElasticSearch/IndexTest.php index 1661143c..e6286ebc 100644 --- a/tests/Unit/ElasticSearch/IndexTest.php +++ b/tests/Unit/ElasticSearch/IndexTest.php @@ -3,9 +3,9 @@ namespace Tests\Unit\ElasticSearch; use App\Product; -use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use Tests\TestCase; use Matchish\ScoutElasticSearch\ElasticSearch\Index; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; class IndexTest extends TestCase { diff --git a/tests/Unit/Jobs/ImportStagesTest.php b/tests/Unit/Jobs/ImportStagesTest.php index 0101abe1..4a1f5f6f 100644 --- a/tests/Unit/Jobs/ImportStagesTest.php +++ b/tests/Unit/Jobs/ImportStagesTest.php @@ -3,7 +3,6 @@ namespace Tests\Unit\Jobs; use App\Product; -use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; use Tests\TestCase; use Matchish\ScoutElasticSearch\Jobs\ImportStages; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; @@ -11,6 +10,7 @@ use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex; use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex; +use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; class ImportStagesTest extends TestCase { From 681f06cee668395a1d60a9036c5297dc6421d3f6 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 10 Aug 2019 09:05:45 +0300 Subject: [PATCH 15/32] Refactor --- src/Console/Commands/ImportSourceFactory.php | 8 -------- src/{Console/Commands => Database/Scopes}/ChunkScope.php | 3 ++- src/ElasticSearch/Index.php | 2 +- src/Jobs/Import.php | 2 +- src/Jobs/ImportStages.php | 2 +- src/Jobs/Stages/CleanUp.php | 2 +- src/Jobs/Stages/CreateWriteIndex.php | 2 +- src/Jobs/Stages/PullFromSource.php | 2 +- src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php | 2 +- .../Commands => Searchable}/DefaultImportSource.php | 6 +++++- .../DefaultImportSourceFactory.php | 3 ++- src/{Console/Commands => Searchable}/ImportSource.php | 2 +- src/Searchable/ImportSourceFactory.php | 8 ++++++++ tests/Integration/Jobs/ImportTest.php | 2 +- tests/Integration/Jobs/Stages/CleanUpTest.php | 2 +- tests/Integration/Jobs/Stages/CreateWriteIndexTest.php | 2 +- tests/Integration/Jobs/Stages/PullFromSourceTest.php | 2 +- .../Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php | 2 +- tests/Unit/ElasticSearch/IndexTest.php | 2 +- tests/Unit/Jobs/ImportStagesTest.php | 2 +- 20 files changed, 32 insertions(+), 26 deletions(-) rename src/{Console/Commands => Database/Scopes}/ChunkScope.php (95%) rename src/{Console/Commands => Searchable}/DefaultImportSource.php (96%) rename src/{Console/Commands => Searchable}/DefaultImportSourceFactory.php (78%) rename src/{Console/Commands => Searchable}/ImportSource.php (80%) create mode 100644 src/Searchable/ImportSourceFactory.php diff --git a/src/Console/Commands/ImportSourceFactory.php b/src/Console/Commands/ImportSourceFactory.php index 559f8f43..e69de29b 100644 --- a/src/Console/Commands/ImportSourceFactory.php +++ b/src/Console/Commands/ImportSourceFactory.php @@ -1,8 +0,0 @@ - Date: Sat, 10 Aug 2019 09:07:51 +0300 Subject: [PATCH 16/32] Refactor --- src/ScoutElasticSearchServiceProvider.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ScoutElasticSearchServiceProvider.php b/src/ScoutElasticSearchServiceProvider.php index 1cf36775..b385e997 100644 --- a/src/ScoutElasticSearchServiceProvider.php +++ b/src/ScoutElasticSearchServiceProvider.php @@ -8,11 +8,11 @@ use Laravel\Scout\EngineManager; use Illuminate\Support\ServiceProvider; use Laravel\Scout\ScoutServiceProvider; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; use Matchish\ScoutElasticSearch\Console\Commands\FlushCommand; use Matchish\ScoutElasticSearch\Console\Commands\ImportCommand; -use Matchish\ScoutElasticSearch\Console\Commands\ImportSourceFactory; -use Matchish\ScoutElasticSearch\Console\Commands\DefaultImportSourceFactory; final class ScoutElasticSearchServiceProvider extends ServiceProvider { From 8869ca579c51abe212a6253e9cedc2411f980f65 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 10 Aug 2019 09:09:27 +0300 Subject: [PATCH 17/32] Register default import scope --- src/Console/Commands/ImportCommand.php | 6 ++++- src/Database/Scopes/DummyImportScope.php | 22 +++++++++++++++++++ src/Database/Scopes/ImportScope.php | 15 +++++++++++++ src/ScoutElasticSearchServiceProvider.php | 3 +++ src/Searchable/DefaultImportSource.php | 3 +-- src/Searchable/DefaultImportSourceFactory.php | 4 ++-- 6 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 src/Database/Scopes/DummyImportScope.php create mode 100644 src/Database/Scopes/ImportScope.php diff --git a/src/Console/Commands/ImportCommand.php b/src/Console/Commands/ImportCommand.php index 3168288f..98dd59bf 100644 --- a/src/Console/Commands/ImportCommand.php +++ b/src/Console/Commands/ImportCommand.php @@ -7,6 +7,9 @@ use Illuminate\Console\Command; use Matchish\ScoutElasticSearch\Jobs\Import; use Matchish\ScoutElasticSearch\Jobs\QueueableJob; +use Matchish\ScoutElasticSearch\Database\Scopes\ImportScope; +use Matchish\ScoutElasticSearch\Searchable\ImportSource; +use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory; final class ImportCommand extends Command @@ -43,7 +46,8 @@ private function searchableList($argument) private function import($searchable) { $sourceFactory = app(ImportSourceFactory::class); - $source = $sourceFactory::from($searchable); + $defaultScope = app(ImportScope::class); + $source = $sourceFactory::from($searchable, [$defaultScope]); $job = new Import($source); if (config('scout.queue')) { diff --git a/src/Database/Scopes/DummyImportScope.php b/src/Database/Scopes/DummyImportScope.php new file mode 100644 index 00000000..847e384d --- /dev/null +++ b/src/Database/Scopes/DummyImportScope.php @@ -0,0 +1,22 @@ +app->register(ScoutServiceProvider::class); $this->app->bind(ImportSourceFactory::class, DefaultImportSourceFactory::class); + $this->app->bind(ImportScope::class, DummyImportScope::class); $this->registerCommands(); } diff --git a/src/Searchable/DefaultImportSource.php b/src/Searchable/DefaultImportSource.php index 3e57a282..fc65f737 100644 --- a/src/Searchable/DefaultImportSource.php +++ b/src/Searchable/DefaultImportSource.php @@ -65,8 +65,7 @@ public function chunked() return collect($pairs)->map(function ($pair) { [$start, $end] = $pair; $chunkScope = new ChunkScope($start, $end); - - return new static($this->className, [$chunkScope]); + return new static($this->className, array_merge($this->scopes, [$chunkScope])); }); } else { return collect(); diff --git a/src/Searchable/DefaultImportSourceFactory.php b/src/Searchable/DefaultImportSourceFactory.php index 2738cf21..c321182c 100644 --- a/src/Searchable/DefaultImportSourceFactory.php +++ b/src/Searchable/DefaultImportSourceFactory.php @@ -5,8 +5,8 @@ class DefaultImportSourceFactory implements ImportSourceFactory { - public static function from(string $className): ImportSource + public static function from(string $className, array $scopes = []): ImportSource { - return new DefaultImportSource($className); + return new DefaultImportSource($className, $scopes); } } From 23557c16314fe0e905a41f3847a267123e2bc661 Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Sat, 10 Aug 2019 06:34:22 +0000 Subject: [PATCH 18/32] Apply fixes from StyleCI --- src/Console/Commands/ImportCommand.php | 2 +- src/Database/Scopes/ChunkScope.php | 1 - src/Database/Scopes/DummyImportScope.php | 4 +--- src/Database/Scopes/ImportScope.php | 2 -- src/Jobs/Import.php | 2 +- src/Jobs/ImportStages.php | 2 +- src/Jobs/Stages/CreateWriteIndex.php | 2 +- src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php | 2 +- src/ScoutElasticSearchServiceProvider.php | 6 +++--- src/Searchable/DefaultImportSource.php | 3 +-- src/Searchable/DefaultImportSourceFactory.php | 1 - tests/Integration/Jobs/ImportTest.php | 2 +- tests/Integration/Jobs/Stages/CleanUpTest.php | 2 +- tests/Integration/Jobs/Stages/CreateWriteIndexTest.php | 2 +- tests/Integration/Jobs/Stages/PullFromSourceTest.php | 2 +- .../Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php | 2 +- tests/Unit/ElasticSearch/IndexTest.php | 2 +- tests/Unit/Jobs/ImportStagesTest.php | 2 +- 18 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Console/Commands/ImportCommand.php b/src/Console/Commands/ImportCommand.php index 98dd59bf..89e69db8 100644 --- a/src/Console/Commands/ImportCommand.php +++ b/src/Console/Commands/ImportCommand.php @@ -7,8 +7,8 @@ use Illuminate\Console\Command; use Matchish\ScoutElasticSearch\Jobs\Import; use Matchish\ScoutElasticSearch\Jobs\QueueableJob; -use Matchish\ScoutElasticSearch\Database\Scopes\ImportScope; use Matchish\ScoutElasticSearch\Searchable\ImportSource; +use Matchish\ScoutElasticSearch\Database\Scopes\ImportScope; use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory; diff --git a/src/Database/Scopes/ChunkScope.php b/src/Database/Scopes/ChunkScope.php index 8a539c0c..60903e76 100644 --- a/src/Database/Scopes/ChunkScope.php +++ b/src/Database/Scopes/ChunkScope.php @@ -1,6 +1,5 @@ map(function ($pair) { [$start, $end] = $pair; $chunkScope = new ChunkScope($start, $end); + return new static($this->className, array_merge($this->scopes, [$chunkScope])); }); } else { diff --git a/src/Searchable/DefaultImportSourceFactory.php b/src/Searchable/DefaultImportSourceFactory.php index c321182c..eac01cf0 100644 --- a/src/Searchable/DefaultImportSourceFactory.php +++ b/src/Searchable/DefaultImportSourceFactory.php @@ -1,6 +1,5 @@ Date: Mon, 12 Aug 2019 06:55:43 +0300 Subject: [PATCH 19/32] Source factory doesn't need import scope --- src/Console/Commands/ImportCommand.php | 3 +-- src/Database/Scopes/DummyImportScope.php | 20 ------------------- src/ScoutElasticSearchServiceProvider.php | 7 ++----- src/Searchable/DefaultImportSourceFactory.php | 4 ++-- src/Searchable/ImportSourceFactory.php | 2 +- 5 files changed, 6 insertions(+), 30 deletions(-) delete mode 100644 src/Database/Scopes/DummyImportScope.php diff --git a/src/Console/Commands/ImportCommand.php b/src/Console/Commands/ImportCommand.php index 89e69db8..005da686 100644 --- a/src/Console/Commands/ImportCommand.php +++ b/src/Console/Commands/ImportCommand.php @@ -46,8 +46,7 @@ private function searchableList($argument) private function import($searchable) { $sourceFactory = app(ImportSourceFactory::class); - $defaultScope = app(ImportScope::class); - $source = $sourceFactory::from($searchable, [$defaultScope]); + $source = $sourceFactory::from($searchable); $job = new Import($source); if (config('scout.queue')) { diff --git a/src/Database/Scopes/DummyImportScope.php b/src/Database/Scopes/DummyImportScope.php deleted file mode 100644 index 6759eec1..00000000 --- a/src/Database/Scopes/DummyImportScope.php +++ /dev/null @@ -1,20 +0,0 @@ -app->register(ScoutServiceProvider::class); $this->app->bind(ImportSourceFactory::class, DefaultImportSourceFactory::class); - $this->app->bind(ImportScope::class, DummyImportScope::class); $this->registerCommands(); } diff --git a/src/Searchable/DefaultImportSourceFactory.php b/src/Searchable/DefaultImportSourceFactory.php index eac01cf0..992e8aeb 100644 --- a/src/Searchable/DefaultImportSourceFactory.php +++ b/src/Searchable/DefaultImportSourceFactory.php @@ -4,8 +4,8 @@ class DefaultImportSourceFactory implements ImportSourceFactory { - public static function from(string $className, array $scopes = []): ImportSource + public static function from(string $className): ImportSource { - return new DefaultImportSource($className, $scopes); + return new DefaultImportSource($className); } } diff --git a/src/Searchable/ImportSourceFactory.php b/src/Searchable/ImportSourceFactory.php index e061b218..41896299 100644 --- a/src/Searchable/ImportSourceFactory.php +++ b/src/Searchable/ImportSourceFactory.php @@ -4,5 +4,5 @@ interface ImportSourceFactory { - public static function from(string $className, array $scopes = []): ImportSource; + public static function from(string $className): ImportSource; } From 922d65a7cc279afdf271447da5fa8819aeb5624b Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Mon, 19 Aug 2019 05:44:29 +0300 Subject: [PATCH 20/32] Add test import chunked with custom key Fix test import chunked with custom key Add test import chunked with custom key Fix test import chunked with custom key --- tests/Feature/ImportCommandTest.php | 26 +++++++++++++++++++++++++ tests/laravel/app/BookWithCustomKey.php | 25 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/laravel/app/BookWithCustomKey.php diff --git a/tests/Feature/ImportCommandTest.php b/tests/Feature/ImportCommandTest.php index bcb515b0..111101a5 100644 --- a/tests/Feature/ImportCommandTest.php +++ b/tests/Feature/ImportCommandTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature; use App\Book; +use App\BookWithCustomKey; use stdClass; use App\Product; use Tests\IntegrationTestCase; @@ -88,6 +89,31 @@ public function test_import_all_pages(): void $response = $this->elasticsearch->search($params); $this->assertEquals($productsAmount, $response['hits']['total']); } + public function test_import_with_custom_key_all_pages(): void + { + $this->app['config']['scout.key'] = 'title'; + + $dispatcher = Book::getEventDispatcher(); + Book::unsetEventDispatcher(); + + $booksAmount = 10; + + factory(Book::class, $booksAmount)->create(); + + Book::setEventDispatcher($dispatcher); + + Artisan::call('scout:import'); + $params = [ + 'index' => (new BookWithCustomKey())->searchableAs(), + 'body' => [ + 'query' => [ + 'match_all' => new stdClass(), + ], + ], + ]; + $response = $this->elasticsearch->search($params); + $this->assertEquals($booksAmount, $response['hits']['total']); + } public function test_remove_old_index_after_switching_to_new(): void { diff --git a/tests/laravel/app/BookWithCustomKey.php b/tests/laravel/app/BookWithCustomKey.php new file mode 100644 index 00000000..5e5f5c09 --- /dev/null +++ b/tests/laravel/app/BookWithCustomKey.php @@ -0,0 +1,25 @@ + Date: Mon, 19 Aug 2019 05:45:34 +0300 Subject: [PATCH 21/32] Fix pull from source when key is custom --- src/Searchable/DefaultImportSource.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Searchable/DefaultImportSource.php b/src/Searchable/DefaultImportSource.php index 601642a4..338a42c2 100644 --- a/src/Searchable/DefaultImportSource.php +++ b/src/Searchable/DefaultImportSource.php @@ -47,7 +47,7 @@ public function chunked() $cloneQuery = clone $query; $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); - $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck('id'); + $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck($searchable->getKeyName()); $pairs = []; $lastId = null; foreach ($ids as $id) { @@ -91,7 +91,7 @@ private function newQuery() $scopes = $this->scopes; return collect($scopes)->reduce(function ($instance, $scope) { - $instance->withGlobalScope($scope->key(), $scope); + $instance->withGlobalScope(get_class($scope), $scope); return $instance; }, $query); From 2fcadf00a68c7f7d495cade2ac132f71cd58a84c Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 9 Nov 2019 08:50:29 +0200 Subject: [PATCH 22/32] Clean up --- src/Console/Commands/ImportSourceFactory.php | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/Console/Commands/ImportSourceFactory.php diff --git a/src/Console/Commands/ImportSourceFactory.php b/src/Console/Commands/ImportSourceFactory.php deleted file mode 100644 index e69de29b..00000000 From 6f0ad69a3045190d2b8adc58644391a9812992b2 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 9 Nov 2019 09:01:56 +0200 Subject: [PATCH 23/32] Clean up --- src/Console/Commands/ImportCommand.php | 1 - src/Database/Scopes/ImportScope.php | 13 ------------- 2 files changed, 14 deletions(-) delete mode 100644 src/Database/Scopes/ImportScope.php diff --git a/src/Console/Commands/ImportCommand.php b/src/Console/Commands/ImportCommand.php index 005da686..bd5c927a 100644 --- a/src/Console/Commands/ImportCommand.php +++ b/src/Console/Commands/ImportCommand.php @@ -8,7 +8,6 @@ use Matchish\ScoutElasticSearch\Jobs\Import; use Matchish\ScoutElasticSearch\Jobs\QueueableJob; use Matchish\ScoutElasticSearch\Searchable\ImportSource; -use Matchish\ScoutElasticSearch\Database\Scopes\ImportScope; use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory; diff --git a/src/Database/Scopes/ImportScope.php b/src/Database/Scopes/ImportScope.php deleted file mode 100644 index 892dd2f0..00000000 --- a/src/Database/Scopes/ImportScope.php +++ /dev/null @@ -1,13 +0,0 @@ - Date: Sat, 9 Nov 2019 09:19:34 +0200 Subject: [PATCH 24/32] Config vm.max_map_count for local development --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index fecda02e..38faeb17 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,7 @@ help: ## Show this help ---------------: ## --------------- up: ## Start all containers (in background) for development + sudo sysctl -w vm.max_map_count=262144 $(docker_compose_bin) up --no-recreate -d down: ## Stop all started for development containers From c9604111f80dd1c9df299d046e36e8cec243447f Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 9 Nov 2019 09:25:01 +0200 Subject: [PATCH 25/32] Make classes final --- src/Searchable/DefaultImportSource.php | 2 +- src/Searchable/DefaultImportSourceFactory.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Searchable/DefaultImportSource.php b/src/Searchable/DefaultImportSource.php index 338a42c2..b5ae2ae7 100644 --- a/src/Searchable/DefaultImportSource.php +++ b/src/Searchable/DefaultImportSource.php @@ -4,7 +4,7 @@ use Matchish\ScoutElasticSearch\Database\Scopes\ChunkScope; -class DefaultImportSource implements ImportSource +final class DefaultImportSource implements ImportSource { const DEFAULT_CHUNK_SIZE = 500; diff --git a/src/Searchable/DefaultImportSourceFactory.php b/src/Searchable/DefaultImportSourceFactory.php index 992e8aeb..294facb1 100644 --- a/src/Searchable/DefaultImportSourceFactory.php +++ b/src/Searchable/DefaultImportSourceFactory.php @@ -2,7 +2,7 @@ namespace Matchish\ScoutElasticSearch\Searchable; -class DefaultImportSourceFactory implements ImportSourceFactory +final class DefaultImportSourceFactory implements ImportSourceFactory { public static function from(string $className): ImportSource { From fc50c6169e89140299ac20ca2cc2a2ecab8f2acb Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Sat, 9 Nov 2019 09:28:23 +0200 Subject: [PATCH 26/32] Use PageScope instead of ChunkScope It make sense to use ChunkScope on huge datasets or if you run import in parallel --- src/Database/Scopes/PageScope.php | 43 ++++++++++++++++++++++++++ src/Searchable/DefaultImportSource.php | 26 +++------------- 2 files changed, 47 insertions(+), 22 deletions(-) create mode 100644 src/Database/Scopes/PageScope.php diff --git a/src/Database/Scopes/PageScope.php b/src/Database/Scopes/PageScope.php new file mode 100644 index 00000000..810a59f3 --- /dev/null +++ b/src/Database/Scopes/PageScope.php @@ -0,0 +1,43 @@ +page = $page; + $this->perPage = $perPage; + } + + /** + * Apply the scope to a given Eloquent query builder. + * + * @param \Illuminate\Database\Eloquent\Builder $builder + * @param \Illuminate\Database\Eloquent\Model $model + * @return void + */ + public function apply(Builder $builder, Model $model) + { + $builder->forPage($this->page, $this->perPage); + } +} diff --git a/src/Searchable/DefaultImportSource.php b/src/Searchable/DefaultImportSource.php index b5ae2ae7..e8b0a978 100644 --- a/src/Searchable/DefaultImportSource.php +++ b/src/Searchable/DefaultImportSource.php @@ -2,7 +2,7 @@ namespace Matchish\ScoutElasticSearch\Searchable; -use Matchish\ScoutElasticSearch\Database\Scopes\ChunkScope; +use Matchish\ScoutElasticSearch\Database\Scopes\PageScope; final class DefaultImportSource implements ImportSource { @@ -40,30 +40,12 @@ public function searchableAs(): string public function chunked() { $query = $this->newQuery(); - $searchable = $this->model(); $totalSearchables = $query->count(); if ($totalSearchables) { $chunkSize = (int) config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); - $cloneQuery = clone $query; - $cloneQuery->joinSub('SELECT @row :=0, 1 as temp', 'r', 'r.temp', 'r.temp') - ->selectRaw("@row := @row +1 AS rownum, {$searchable->getKeyName()}"); - $ids = \DB::query()->fromSub($cloneQuery, 'ranked')->whereRaw("rownum %{$chunkSize} =0")->pluck($searchable->getKeyName()); - $pairs = []; - $lastId = null; - foreach ($ids as $id) { - if ($lastId) { - $pairs[] = [$lastId, $id]; - } else { - $pairs[] = [null, $id]; - } - $lastId = $id; - } - $pairs[] = [$lastId, null]; - - return collect($pairs)->map(function ($pair) { - [$start, $end] = $pair; - $chunkScope = new ChunkScope($start, $end); - + $totalChunks = (int) ceil($totalSearchables / $chunkSize); + return collect(range(1, $totalChunks))->map(function ($page) use ($chunkSize) { + $chunkScope = new PageScope($page, $chunkSize); return new static($this->className, array_merge($this->scopes, [$chunkScope])); }); } else { From 6f31770604d5583029648726e1ff3ac73d25d034 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Tue, 12 Nov 2019 08:59:09 +0200 Subject: [PATCH 27/32] Add test default import source apply scope to new query --- .../Searchable/DefaultImportSourceTest.php | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/Integration/Searchable/DefaultImportSourceTest.php diff --git a/tests/Integration/Searchable/DefaultImportSourceTest.php b/tests/Integration/Searchable/DefaultImportSourceTest.php new file mode 100644 index 00000000..2a2796f5 --- /dev/null +++ b/tests/Integration/Searchable/DefaultImportSourceTest.php @@ -0,0 +1,49 @@ +states(['iphone', 'promo', 'used'])->create(); + factory(Product::class, $iphonePromoNewAmount)->states(['iphone', 'promo', 'new'])->create(); + + Product::setEventDispatcher($dispatcher); + $source = new DefaultImportSource(Product::class, [new UsedScope()]); + $products = $source->get(); + $this->assertEquals($iphonePromoUsedAmount, $products->count()); + + } +} + +class UsedScope implements Scope { + + /** + * Apply the scope to a given Eloquent query builder. + * + * @param \Illuminate\Database\Eloquent\Builder $builder + * @param \Illuminate\Database\Eloquent\Model $model + * @return void + */ + public function apply(Builder $builder, Model $model) + { + $builder->where('type', 'used'); + } +} From a251599192a87f01edec02c83841694007947a5f Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Tue, 12 Nov 2019 09:22:38 +0200 Subject: [PATCH 28/32] Clean up --- tests/Feature/ImportCommandTest.php | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Feature/ImportCommandTest.php b/tests/Feature/ImportCommandTest.php index dfbf3791..93b693f6 100644 --- a/tests/Feature/ImportCommandTest.php +++ b/tests/Feature/ImportCommandTest.php @@ -9,7 +9,6 @@ use stdClass; use App\Product; use Illuminate\Support\Facades\Artisan; -use stdClass; use Symfony\Component\Console\Output\BufferedOutput; use Tests\IntegrationTestCase; From a390e4f1a4b123e155c0ad64588e1e84e6173089 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Tue, 12 Nov 2019 09:23:17 +0200 Subject: [PATCH 29/32] Resolve conflict --- tests/Integration/Engines/ElasticSearchEngineTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Integration/Engines/ElasticSearchEngineTest.php b/tests/Integration/Engines/ElasticSearchEngineTest.php index fd2f25a5..58a3c0f6 100644 --- a/tests/Integration/Engines/ElasticSearchEngineTest.php +++ b/tests/Integration/Engines/ElasticSearchEngineTest.php @@ -7,6 +7,7 @@ use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Create; use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; use stdClass; use Tests\IntegrationTestCase; @@ -66,7 +67,7 @@ public function test_update_throw_exception_on_elasticsearch_error() return $model; }); - $index = Index::fromSearchable($models->first()); + $index = Index::fromSource(DefaultImportSourceFactory::from(Product::class)); $params = new Create( 'products', $index->config() From 5fb5e956846050ba8fb554c46387f0f39c31d081 Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Tue, 12 Nov 2019 18:38:55 +0000 Subject: [PATCH 30/32] Apply fixes from StyleCI --- src/Database/Scopes/ChunkScope.php | 2 +- src/Database/Scopes/PageScope.php | 1 - src/ElasticSearch/EloquentHitsIteratorAggregate.php | 2 +- src/Jobs/ImportStages.php | 6 +++--- src/Jobs/Stages/CleanUp.php | 2 +- src/Jobs/Stages/CreateWriteIndex.php | 6 +++--- src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php | 2 +- src/ScoutElasticSearchServiceProvider.php | 8 ++++---- src/Searchable/DefaultImportSource.php | 2 ++ tests/Feature/ImportCommandTest.php | 3 ++- tests/Integration/Jobs/ImportTest.php | 4 ++-- tests/Integration/Jobs/Stages/CleanUpTest.php | 4 ++-- tests/Integration/Jobs/Stages/CreateWriteIndexTest.php | 2 +- tests/Integration/Jobs/Stages/PullFromSourceTest.php | 4 ++-- .../Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php | 2 +- tests/Integration/Searchable/DefaultImportSourceTest.php | 8 ++------ tests/Unit/ElasticSearch/IndexTest.php | 2 +- tests/Unit/Jobs/ImportStagesTest.php | 8 ++++---- 18 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/Database/Scopes/ChunkScope.php b/src/Database/Scopes/ChunkScope.php index 60903e76..68835cb4 100644 --- a/src/Database/Scopes/ChunkScope.php +++ b/src/Database/Scopes/ChunkScope.php @@ -2,9 +2,9 @@ namespace Matchish\ScoutElasticSearch\Database\Scopes; +use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Scope; -use Illuminate\Database\Eloquent\Builder; class ChunkScope implements Scope { diff --git a/src/Database/Scopes/PageScope.php b/src/Database/Scopes/PageScope.php index 810a59f3..debae824 100644 --- a/src/Database/Scopes/PageScope.php +++ b/src/Database/Scopes/PageScope.php @@ -8,7 +8,6 @@ class PageScope implements Scope { - /** * @var int */ diff --git a/src/ElasticSearch/EloquentHitsIteratorAggregate.php b/src/ElasticSearch/EloquentHitsIteratorAggregate.php index d5082e7d..d42b38d1 100644 --- a/src/ElasticSearch/EloquentHitsIteratorAggregate.php +++ b/src/ElasticSearch/EloquentHitsIteratorAggregate.php @@ -2,10 +2,10 @@ namespace Matchish\ScoutElasticSearch\ElasticSearch; -use Traversable; use IteratorAggregate; use Laravel\Scout\Builder; use Laravel\Scout\Searchable; +use Traversable; /** * @internal diff --git a/src/Jobs/ImportStages.php b/src/Jobs/ImportStages.php index 18aaebc5..576486d6 100644 --- a/src/Jobs/ImportStages.php +++ b/src/Jobs/ImportStages.php @@ -5,11 +5,11 @@ use Illuminate\Support\Collection; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; -use Matchish\ScoutElasticSearch\Searchable\ImportSource; -use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex; -use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex; +use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; +use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex; use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex; +use Matchish\ScoutElasticSearch\Searchable\ImportSource; class ImportStages extends Collection { diff --git a/src/Jobs/Stages/CleanUp.php b/src/Jobs/Stages/CleanUp.php index e5b12e8c..ff11ecb3 100644 --- a/src/Jobs/Stages/CleanUp.php +++ b/src/Jobs/Stages/CleanUp.php @@ -4,9 +4,9 @@ use Elasticsearch\Client; use Elasticsearch\Common\Exceptions\Missing404Exception; -use Matchish\ScoutElasticSearch\Searchable\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get as GetAliasParams; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Delete as DeleteIndexParams; +use Matchish\ScoutElasticSearch\Searchable\ImportSource; /** * @internal diff --git a/src/Jobs/Stages/CreateWriteIndex.php b/src/Jobs/Stages/CreateWriteIndex.php index 396b96f3..8c742d45 100644 --- a/src/Jobs/Stages/CreateWriteIndex.php +++ b/src/Jobs/Stages/CreateWriteIndex.php @@ -3,11 +3,11 @@ namespace Matchish\ScoutElasticSearch\Jobs\Stages; use Elasticsearch\Client; -use Matchish\ScoutElasticSearch\ElasticSearch\Index; -use Matchish\ScoutElasticSearch\Searchable\ImportSource; -use Matchish\ScoutElasticSearch\ElasticSearch\WriteAlias; use Matchish\ScoutElasticSearch\ElasticSearch\DefaultAlias; +use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Create; +use Matchish\ScoutElasticSearch\ElasticSearch\WriteAlias; +use Matchish\ScoutElasticSearch\Searchable\ImportSource; /** * @internal diff --git a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php index 7181b9eb..9257a4c0 100644 --- a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php +++ b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php @@ -4,9 +4,9 @@ use Elasticsearch\Client; use Matchish\ScoutElasticSearch\ElasticSearch\Index; -use Matchish\ScoutElasticSearch\Searchable\ImportSource; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Update; +use Matchish\ScoutElasticSearch\Searchable\ImportSource; /** * @internal diff --git a/src/ScoutElasticSearchServiceProvider.php b/src/ScoutElasticSearchServiceProvider.php index b385e997..be00314b 100644 --- a/src/ScoutElasticSearchServiceProvider.php +++ b/src/ScoutElasticSearchServiceProvider.php @@ -5,14 +5,14 @@ namespace Matchish\ScoutElasticSearch; use Elasticsearch\Client; -use Laravel\Scout\EngineManager; use Illuminate\Support\ServiceProvider; +use Laravel\Scout\EngineManager; use Laravel\Scout\ScoutServiceProvider; -use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; -use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; -use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; use Matchish\ScoutElasticSearch\Console\Commands\FlushCommand; use Matchish\ScoutElasticSearch\Console\Commands\ImportCommand; +use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; final class ScoutElasticSearchServiceProvider extends ServiceProvider { diff --git a/src/Searchable/DefaultImportSource.php b/src/Searchable/DefaultImportSource.php index e8b0a978..220221a6 100644 --- a/src/Searchable/DefaultImportSource.php +++ b/src/Searchable/DefaultImportSource.php @@ -44,8 +44,10 @@ public function chunked() if ($totalSearchables) { $chunkSize = (int) config('scout.chunk.searchable', self::DEFAULT_CHUNK_SIZE); $totalChunks = (int) ceil($totalSearchables / $chunkSize); + return collect(range(1, $totalChunks))->map(function ($page) use ($chunkSize) { $chunkScope = new PageScope($page, $chunkSize); + return new static($this->className, array_merge($this->scopes, [$chunkScope])); }); } else { diff --git a/tests/Feature/ImportCommandTest.php b/tests/Feature/ImportCommandTest.php index 93b693f6..3d98a254 100644 --- a/tests/Feature/ImportCommandTest.php +++ b/tests/Feature/ImportCommandTest.php @@ -6,9 +6,9 @@ use App\Book; use App\BookWithCustomKey; -use stdClass; use App\Product; use Illuminate\Support\Facades\Artisan; +use stdClass; use Symfony\Component\Console\Output\BufferedOutput; use Tests\IntegrationTestCase; @@ -91,6 +91,7 @@ public function test_import_all_pages(): void $response = $this->elasticsearch->search($params); $this->assertEquals($productsAmount, $response['hits']['total']); } + public function test_import_with_custom_key_all_pages(): void { $this->app['config']['scout.key'] = 'title'; diff --git a/tests/Integration/Jobs/ImportTest.php b/tests/Integration/Jobs/ImportTest.php index cd0fe409..7a311593 100644 --- a/tests/Integration/Jobs/ImportTest.php +++ b/tests/Integration/Jobs/ImportTest.php @@ -3,12 +3,12 @@ namespace Tests\Integration\Jobs; use App\Product; -use Tests\IntegrationTestCase; use Illuminate\Console\OutputStyle; use Matchish\ScoutElasticSearch\Jobs\Import; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Tests\Fixtures\DummyOutput; -use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Tests\IntegrationTestCase; class ImportTest extends IntegrationTestCase { diff --git a/tests/Integration/Jobs/Stages/CleanUpTest.php b/tests/Integration/Jobs/Stages/CleanUpTest.php index d43fed5d..fca0d015 100644 --- a/tests/Integration/Jobs/Stages/CleanUpTest.php +++ b/tests/Integration/Jobs/Stages/CleanUpTest.php @@ -2,11 +2,11 @@ namespace Tests\Integration\Jobs\Stages; -use stdClass; use App\Product; -use Tests\IntegrationTestCase; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use stdClass; +use Tests\IntegrationTestCase; class CleanUpTest extends IntegrationTestCase { diff --git a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php index e416dd7d..dd54ea42 100644 --- a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php +++ b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php @@ -6,10 +6,10 @@ use App\Product; use Elasticsearch\Client; -use Tests\IntegrationTestCase; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex; use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Tests\IntegrationTestCase; final class CreateWriteIndexTest extends IntegrationTestCase { diff --git a/tests/Integration/Jobs/Stages/PullFromSourceTest.php b/tests/Integration/Jobs/Stages/PullFromSourceTest.php index 2acb4b0a..8cbe31f6 100644 --- a/tests/Integration/Jobs/Stages/PullFromSourceTest.php +++ b/tests/Integration/Jobs/Stages/PullFromSourceTest.php @@ -4,11 +4,11 @@ namespace Tests\Integration\Jobs\Stages; -use stdClass; use App\Product; -use Tests\IntegrationTestCase; use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use stdClass; +use Tests\IntegrationTestCase; final class PullFromSourceTest extends IntegrationTestCase { diff --git a/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php b/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php index 7e0b4381..fd23af70 100644 --- a/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php +++ b/tests/Integration/Jobs/Stages/SwitchToNewAndRemoveOldIndexTest.php @@ -6,8 +6,8 @@ use App\Product; use Matchish\ScoutElasticSearch\ElasticSearch\Index; -use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; use stdClass; use Tests\IntegrationTestCase; diff --git a/tests/Integration/Searchable/DefaultImportSourceTest.php b/tests/Integration/Searchable/DefaultImportSourceTest.php index 2a2796f5..22718a25 100644 --- a/tests/Integration/Searchable/DefaultImportSourceTest.php +++ b/tests/Integration/Searchable/DefaultImportSourceTest.php @@ -2,7 +2,6 @@ namespace Tests\Integration\Searchable; - use App\Product; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Model; @@ -12,7 +11,6 @@ class DefaultImportSourceTest extends TestCase { - public function test_new_query_has_injected_scopes() { $dispatcher = Product::getEventDispatcher(); @@ -21,7 +19,6 @@ public function test_new_query_has_injected_scopes() $iphonePromoUsedAmount = rand(1, 5); $iphonePromoNewAmount = rand(6, 10); - factory(Product::class, $iphonePromoUsedAmount)->states(['iphone', 'promo', 'used'])->create(); factory(Product::class, $iphonePromoNewAmount)->states(['iphone', 'promo', 'new'])->create(); @@ -29,12 +26,11 @@ public function test_new_query_has_injected_scopes() $source = new DefaultImportSource(Product::class, [new UsedScope()]); $products = $source->get(); $this->assertEquals($iphonePromoUsedAmount, $products->count()); - } } -class UsedScope implements Scope { - +class UsedScope implements Scope +{ /** * Apply the scope to a given Eloquent query builder. * diff --git a/tests/Unit/ElasticSearch/IndexTest.php b/tests/Unit/ElasticSearch/IndexTest.php index 97cec71e..882fac43 100644 --- a/tests/Unit/ElasticSearch/IndexTest.php +++ b/tests/Unit/ElasticSearch/IndexTest.php @@ -3,9 +3,9 @@ namespace Tests\Unit\ElasticSearch; use App\Product; -use Tests\TestCase; use Matchish\ScoutElasticSearch\ElasticSearch\Index; use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Tests\TestCase; class IndexTest extends TestCase { diff --git a/tests/Unit/Jobs/ImportStagesTest.php b/tests/Unit/Jobs/ImportStagesTest.php index 77d32cea..cd055c71 100644 --- a/tests/Unit/Jobs/ImportStagesTest.php +++ b/tests/Unit/Jobs/ImportStagesTest.php @@ -3,14 +3,14 @@ namespace Tests\Unit\Jobs; use App\Product; -use Tests\TestCase; use Matchish\ScoutElasticSearch\Jobs\ImportStages; use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp; -use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex; -use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex; -use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource; +use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex; use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex; +use Matchish\ScoutElasticSearch\Searchable\DefaultImportSourceFactory; +use Tests\TestCase; class ImportStagesTest extends TestCase { From a953fd398cac7b4448c4cdc2b203226157c06142 Mon Sep 17 00:00:00 2001 From: Sergey Shliakhov Date: Wed, 13 Nov 2019 07:34:11 +0200 Subject: [PATCH 31/32] Bump version --- CHANGELOG.md | 5 +++++ README.md | 39 +++++++++++++++++++++++++++++++++++++++ composer.json | 2 +- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 139fd2cd..f899acd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/) ## [Unreleased] +## [2.1.0] - 2019-11-13 +### Added +- Import source factory +- Using global scopes only for import + ## [2.0.4] - 2019-11-10 ### Fixed - Throw more descriptive exception if there are elasticsearch errors on update diff --git a/README.md b/README.md index 6e4d8637..d6b3b8ae 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ If you need any help, [stack overflow](https://stackoverflow.com/questions/tagge - [Search amongst multiple models](#search-amongst-multiple-models) - [**Zero downtime** reimport](#zero-downtime-reimport) - it’s a breeze to import data in production. +- [Eager load for relations](#eager-load) - it’s a breeze to import data in production. - Elasticsearch **7.0** ready - Use [elasticsearch-7](https://github.com/matchish/laravel-scout-elasticsearch/tree/elasticsearch-7) branch instead. - Import all searchable models at once. - A fully configurable mapping for each model. @@ -95,6 +96,44 @@ For index `products` it will be And for default settings `elasticsearch.indices.settings.default` +### Eager load +To speed up import you can eager load relations on import only using global scopes. + +You should configure inject `ImportSourceFactory` in your service provider(`register` method) +``` +use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; +... +public function register(): void +{ +$this->app->bind(ImportSourceFactory::class, MyImportSourceFactory::class); +``` +Here an example of `MyImportSourceFactory` +``` +namespace Matchish\ScoutElasticSearch\Searchable; + +final class MyImportSourceFactory implements ImportSourceFactory +{ + public static function from(string $className): ImportSource + { + return new DefaultImportSource($className, [new WithCommentsScope()]); + } +} + +class WithCommentsScope implements Scope { + + /** + * Apply the scope to a given Eloquent query builder. + * + * @param \Illuminate\Database\Eloquent\Builder $builder + * @param \Illuminate\Database\Eloquent\Model $model + * @return void + */ + public function apply(Builder $builder, Model $model) + { + $builder->with('comments'); + } +} +``` ### Zero downtime reimport While working in production, to keep your existing search experience available while reimporting your data, you also can use `scout:import` Artisan command: diff --git a/composer.json b/composer.json index e835a7f8..4624687d 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "matchish/laravel-scout-elasticsearch", "description": "This package extends Laravel Scout adding full power of ElasticSearch", - "version": "2.0.4", + "version": "2.1.0", "keywords": [ "laravel", "scout", From 6bda7b86ebe499de5ce19c2b040803609f3b50c0 Mon Sep 17 00:00:00 2001 From: Sergey Shlyakhov Date: Wed, 13 Nov 2019 08:06:17 +0200 Subject: [PATCH 32/32] Polish --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d6b3b8ae..a2081150 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ If you need any help, [stack overflow](https://stackoverflow.com/questions/tagge - [Search amongst multiple models](#search-amongst-multiple-models) - [**Zero downtime** reimport](#zero-downtime-reimport) - it’s a breeze to import data in production. -- [Eager load for relations](#eager-load) - it’s a breeze to import data in production. +- [Eager load relations](#eager-load) - speed up your import. - Elasticsearch **7.0** ready - Use [elasticsearch-7](https://github.com/matchish/laravel-scout-elasticsearch/tree/elasticsearch-7) branch instead. - Import all searchable models at once. - A fully configurable mapping for each model. @@ -97,9 +97,9 @@ And for default settings `elasticsearch.indices.settings.default` ### Eager load -To speed up import you can eager load relations on import only using global scopes. +To speed up import you can eager load relations on import using global scopes. -You should configure inject `ImportSourceFactory` in your service provider(`register` method) +You should configure `ImportSourceFactory` in your service provider(`register` method) ``` use Matchish\ScoutElasticSearch\Searchable\ImportSourceFactory; ... @@ -107,7 +107,7 @@ public function register(): void { $this->app->bind(ImportSourceFactory::class, MyImportSourceFactory::class); ``` -Here an example of `MyImportSourceFactory` +Here is an example of `MyImportSourceFactory` ``` namespace Matchish\ScoutElasticSearch\Searchable; @@ -115,6 +115,7 @@ final class MyImportSourceFactory implements ImportSourceFactory { public static function from(string $className): ImportSource { + //Add all required scopes return new DefaultImportSource($className, [new WithCommentsScope()]); } }