diff --git a/.gitignore b/.gitignore index d7e4fa0..11801ab 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ composer.lock .DS_Store Thumbs.db /phpunit.xml -/build \ No newline at end of file +/build +/.idea \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 94af623..278e61f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,14 @@ language: php php: - 7.0 - 5.6 + +branches: + only: + - master + - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ before_script: - travis_retry composer self-update - travis_retry composer install --prefer-source --no-interaction --dev -script: vendor/phpunit/phpunit/phpunit --verbose \ No newline at end of file +script: vendor/phpunit/phpunit/phpunit --verbose diff --git a/README.md b/README.md index 2840eff..513fb04 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Laravel Scout Elasticsearch Driver -[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md) +[![Software License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md) +[![Travis](https://img.shields.io/travis/thomasjsn/laravel-scout-elastic.svg)](https://travis-ci.org/thomasjsn/laravel-scout-elastic) +[![Packagist](https://img.shields.io/packagist/v/thomasjsn/laravel-scout-elastic.svg)](https://packagist.org/packages/thomasjsn/laravel-scout-elastic) This package makes is the [Elasticsearch](https://www.elastic.co/products/elasticsearch) driver for Laravel Scout. @@ -16,7 +18,7 @@ This package makes is the [Elasticsearch](https://www.elastic.co/products/elasti You can install the package via composer: ``` bash -composer require tamayo/laravel-scout-elastic +composer require thomasjsn/laravel-scout-elastic ``` You must add the Scout service provider and the package service provider in your app.php config: @@ -31,10 +33,14 @@ You must add the Scout service provider and the package service provider in your ], ``` -### Setting up Elasticsearch configuration -You must have a Elasticsearch server up and running with the index you want to use created +Then you should publish the Elasticsearch configuration using the `vendor:publish` Artisan command. -If you need help with this please refer to the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) +``` +php artisan vendor:publish --provider="ScoutEngines\Elasticsearch\ElasticsearchProvider" +``` + +### Setting up Elasticsearch configuration +You must have a Elasticsearch server up and running, indices can be created with an Artisan command; see below. After you've published the Laravel Scout package configuration: @@ -42,23 +48,133 @@ After you've published the Laravel Scout package configuration: // config/scout.php // Set your driver to elasticsearch 'driver' => env('SCOUT_DRIVER', 'elasticsearch'), +``` + +## Usage + +### Creating Elasticsearch indexes with proper mapping + +You may define custom mappings for Elasticsearch fields in the config. See examples in the [config file](config/elasticsearch.php). +If you prefer storing mappings in models, you may create a static public method `mapping()` in each particular model: + +```php +class Article extends Model +{ + // ... + public static function mapping() { + return [ + 'location' => [ + 'type' => 'geo_point' + ], + ]; + } + // ... +} +``` +And then use it in the config file: +```php + 'indices' => [ -... - 'elasticsearch' => [ - 'index' => env('ELASTICSEARCH_INDEX', 'laravel'), - 'hosts' => [ - env('ELASTICSEARCH_HOST', 'http://localhost'), + 'realestate' => [ + 'settings' => [ + "number_of_shards" => 1, + "number_of_replicas" => 0, + ], + 'mappings' => [ + 'articles' => \App\Article::mapping(), ], ], -... + ] ``` +The document type, in this example `articles` must match `searchableAs()` for the respective model. -## Usage +Elasticsearch can set default types to model fields on the first insert if you do not explicitly define them. +However; sometimes the defaults are not what you're looking for, or you need to define additional mapping properties. + +In that case, we strongly recommend creating indices with proper mappings before inserting any data. +For that purpose, there is an Artisan command, called `elastic:make-indices {index}` which creates an index based on +the settings in your configuration file. + +To create all indices from your config just ignore the `{index}` parameter and run: + +``` +php artisan elastic:make-indices +``` + +If the index exists you will be asked if you want to delete and recreate it, or you can use the `--force` flag. + +To get information about your existing Elasticsearch indices you may want to use the following command: + +``` +php artisan elastic:indices +``` + +### Indexing data + +You may follow instructions from the [official Laravel Scout documentation](https://laravel.com/docs/5.3/scout) +to index your data. + +### Search + +The package supports everything that is provided by Laravel Scout. + +The Scout `search` method used the default query method defined in the config file. + +Sorting with `orderBy()` method: + +```php +$articles = Article::search($keywords) + ->orderBy('id', 'desc') + ->get(); +``` + +#### Elastic specific + +However, to use the extra Elasticsearch features included in this package, use trait `ElasticSearchable` +by adding it to your model instead of `Searchable`: + +```php +class Article extends Model +{ + // use Searchable; + use ElasticSearchable; + // ... +} +``` + +The package features: + +1) The `elasticSearch` method, `elasticSearch($method, $query, array $params = null)`: + +```php +$articles = Article::elasticSearch('multi_match', $q, [ + 'fields' => ['title', 'content', 'tags'], + 'fuzziness' => 'auto', + 'prefix_length' => 2, + 'operator' => 'AND' +])->get(); +``` + +Parameters are taken from the configuration, for the specific query method, if not supplied. But you may override them. + +2) A separate Elasticsearch index for each model. + +If you have defined several indices in your [config file](config/elasticsearch.php), +you may choose which index a model belongs to by overriding `searchableWithin()` method in your model: + +```php +public function searchableWithin() +{ + return 'foobar'; +} +``` + +If you do not override `searchableWithin()` in your model, the first index from the config will be used. -Now you can use Laravel Scout as described in the [official documentation](https://laravel.com/docs/5.3/scout) ## Credits - [Erick Tamayo](https://github.com/ericktamayo) +- [Thomas Jensen](https://github.com/thomasjsn) - [All Contributors](../../contributors) ## License diff --git a/composer.json b/composer.json index 4e09a5c..f717bf4 100644 --- a/composer.json +++ b/composer.json @@ -1,6 +1,8 @@ { - "name": "tamayo/laravel-scout-elastic", + "name": "thomasjsn/laravel-scout-elastic", "description": "Elastic Driver for Laravel Scout", + "homepage": "https://github.com/thomasjsn/laravel-scout-elastic", + "license": "MIT", "keywords": ["laravel", "scout", "elasticsearch", "elastic"], "require": { "php": ">=5.6.4", diff --git a/config/elasticsearch.php b/config/elasticsearch.php new file mode 100644 index 0000000..e1649be --- /dev/null +++ b/config/elasticsearch.php @@ -0,0 +1,93 @@ + [ + env('ELASTICSEARCH_HOST', 'http://localhost'), + ], + + /* + |-------------------------------------------------------------------------- + | Queries and query parameters + |-------------------------------------------------------------------------- + | + | Here you can specify different search methods, and their parameters. + | The scout "search" method uses the default query type, with its parameters. + | If you use the "elasticSearch" method you can specify the query type and + | override the search parameters when performing the search. + | + */ + + 'queries' => [ + 'default' => 'query_string', + + 'query_string' => [ + 'default_operator' => "AND" + ], + 'multi_match' => [ + 'fields' => '_all', + 'fuzziness' => 'auto' + ] + ], + + /* + |-------------------------------------------------------------------------- + | Elasticsearch indices + |-------------------------------------------------------------------------- + | + | Here you can define your indices, with separate settings and mappings. + | You can choose which index a model belongs to my overriding the + | searchableWithin() method. A model will, by default, belong to the first + | index listed here. + | + | You may specify your mappings in the model if you like that approach, + | just make a static method, e.g. mapping() and refer to it here, like: + | + | 'mappings' => [ + | 'articles' => \App\Article::mapping() + | ] + | + */ + + 'indices' => [ + + 'laravel' => [ + 'settings' => [ + "number_of_shards" => 1, + "number_of_replicas" => 0, + ], + 'mappings' => [ + 'articles' => [ + 'title' => [ + 'type' => 'string' + ] + ] + ] + ], + + 'another_index' => [ + 'settings' => [ + "number_of_shards" => 1, + "number_of_replicas" => 0, + ], + 'mappings' => [ + 'articles' => [ + 'title' => [ + 'type' => 'string' + ] + ] + ] + ] + + ] + +]; diff --git a/src/Console/ElasticIndicesCommand.php b/src/Console/ElasticIndicesCommand.php new file mode 100644 index 0000000..62dbd48 --- /dev/null +++ b/src/Console/ElasticIndicesCommand.php @@ -0,0 +1,56 @@ +setHosts($host)->build(); + + $indices = $client->cat()->indices(); + + if(count($indices) > 0) { + $headers = array_keys(current($indices)); + $this->table($headers, $indices); + } else { + $this->warn('No indices found.'); + } + + } + +} diff --git a/src/Console/ElasticMakeIndicesCommand.php b/src/Console/ElasticMakeIndicesCommand.php new file mode 100644 index 0000000..2ba426c --- /dev/null +++ b/src/Console/ElasticMakeIndicesCommand.php @@ -0,0 +1,98 @@ +setHosts($host)->build(); + + $indices = ! is_null($this->argument('index')) ? + [$this->argument('index')] : array_keys(config('elasticsearch.indices')); + + foreach ($indices as $index) { + + $indexConfig = config("elasticsearch.indices.{$index}"); + + if(is_null($indexConfig)) { + $this->error("Config for index \"{$index}\" not found, skipping..."); + continue; + } + + // Delete index if it already exists + if ($client->indices()->exists(['index' => $index])) { + if ($this->option('force') || $this->confirm("Index \"{$index}\" exists, delete and recreate?")) { + $this->warn("Index \"{$index}\" exists, deleting!"); + $client->indices()->delete(['index' => $index]); + } else { + $this->line("Skipping index: \"{$index}\""); + continue; + } + } + + // Create index with settings from config file + $this->info("Creating index: {$index}"); + $client->indices()->create([ + 'index' => $index, + 'body' => [ + "settings" => $indexConfig['settings'] + ] + ]); + + if (! isset($indexConfig['mappings'])) { + continue; + } + + foreach ($indexConfig['mappings'] as $type => $mapping) { + + // Create mapping for type, from config file + $this->info("- Creating mapping for: {$type}"); + $client->indices()->putMapping([ + 'index' => $index, + 'type' => $type, + 'body' => [ + 'properties' => $mapping + ] + ]); + } + + } + + } + +} diff --git a/src/Contracts/IsElasticSearchable.php b/src/Contracts/IsElasticSearchable.php new file mode 100644 index 0000000..0b0a09d --- /dev/null +++ b/src/Contracts/IsElasticSearchable.php @@ -0,0 +1,20 @@ +elasticQuery = [ + 'method' => $method, + 'params' => $params + ]; + + return new ScoutBuilder($model, $query); + } + +} diff --git a/src/ElasticsearchEngine.php b/src/ElasticsearchEngine.php index 7b6d23f..d9de825 100644 --- a/src/ElasticsearchEngine.php +++ b/src/ElasticsearchEngine.php @@ -6,27 +6,31 @@ use Laravel\Scout\Engines\Engine; use Elasticsearch\Client as Elastic; use Illuminate\Database\Eloquent\Collection; -use Illuminate\Support\Collection as BaseCollection; class ElasticsearchEngine extends Engine { /** - * Index where the models will be saved. + * @var Elastic Client + */ + protected $elastic; + + /** + * Default query and query parameters from scout config. * - * @var string + * @var array */ - protected $index; + protected $queryConfig; /** * Create a new engine instance. * - * @param \Elasticsearch\Client $elastic - * @return void + * @param \Elasticsearch\Client $elastic + * @param $queryConfig */ - public function __construct(Elastic $elastic, $index) + public function __construct(Elastic $elastic, $queryConfig) { $this->elastic = $elastic; - $this->index = $index; + $this->queryConfig = $queryConfig; } /** @@ -44,7 +48,7 @@ public function update($models) $params['body'][] = [ 'update' => [ '_id' => $model->getKey(), - '_index' => $this->index, + '_index' => $model->searchableWithin(), '_type' => $model->searchableAs(), ] ]; @@ -72,7 +76,7 @@ public function delete($models) $params['body'][] = [ 'delete' => [ '_id' => $model->getKey(), - '_index' => $this->index, + '_index' => $model->searchableWithin(), '_type' => $model->searchableAs(), ] ]; @@ -91,6 +95,7 @@ public function search(Builder $builder) { return $this->performSearch($builder, array_filter([ 'numericFilters' => $this->filters($builder), + 'sorting' => $this->sorting($builder), 'size' => $builder->limit, ])); } @@ -107,11 +112,12 @@ public function paginate(Builder $builder, $perPage, $page) { $result = $this->performSearch($builder, [ 'numericFilters' => $this->filters($builder), + 'sorting' => $this->sorting($builder), 'from' => (($page * $perPage) - $perPage), 'size' => $perPage, ]); - $result['nbPages'] = $result['hits']['total']/$perPage; + $result['nbPages'] = $result['hits']['total']/$perPage; return $result; } @@ -125,15 +131,31 @@ public function paginate(Builder $builder, $perPage, $page) */ protected function performSearch(Builder $builder, array $options = []) { + $queryMethod = isset($builder->model->elasticQuery['method']) ? + $builder->model->elasticQuery['method'] : $this->queryConfig['default']; + + $queryParams = isset($builder->model->elasticQuery['params']) ? + $builder->model->elasticQuery['params'] : $this->queryConfig[$queryMethod]; + $params = [ - 'index' => $this->index, + 'index' => $builder->model->searchableWithin(), 'type' => $builder->model->searchableAs(), 'body' => [ 'query' => [ 'bool' => [ - 'must' => [['query_string' => [ 'query' => "*{$builder->query}*"]]] + 'must' => [ + [ + $queryMethod => array_merge([ + 'query' => "{$builder->query}" + ], $queryParams) + ] + ] ] - ] + ], + 'sort' => [ + '_score' + ], + 'track_scores' => true, ] ]; @@ -146,8 +168,13 @@ protected function performSearch(Builder $builder, array $options = []) } if (isset($options['numericFilters']) && count($options['numericFilters'])) { - $params['body']['query']['bool']['must'] = array_merge($params['body']['query']['bool']['must'], - $options['numericFilters']); + $params['body']['query']['bool']['filter'] = $options['numericFilters']; + } + + // Sorting + if(isset($options['sorting']) && count($options['sorting'])) { + $params['body']['sort'] = array_merge($params['body']['sort'], + $options['sorting']); } return $this->elastic->search($params); @@ -162,7 +189,18 @@ protected function performSearch(Builder $builder, array $options = []) protected function filters(Builder $builder) { return collect($builder->wheres)->map(function ($value, $key) { - return ['match_phrase' => [$key => $value]]; + return ['term' => [$key => $value]]; + })->values()->all(); + } + + /** + * @param Builder $builder + * @return array + */ + protected function sorting(Builder $builder) + { + return collect($builder->orders)->map(function ($value, $key) { + return [array_get($value, 'column') => ['order' => array_get($value, 'direction')]]; })->values()->all(); } @@ -191,15 +229,19 @@ public function map($results, $model) } $keys = collect($results['hits']['hits']) - ->pluck('_id')->values()->all(); + ->pluck('_id')->values()->all(); $models = $model->whereIn( $model->getKeyName(), $keys )->get()->keyBy($model->getKeyName()); return collect($results['hits']['hits'])->map(function ($hit) use ($model, $models) { - return $models[$hit['_id']]; - }); + $key = $hit['_id']; + + if (isset($models[$key])) { + return $models[$key]; + } + })->filter(); } /** diff --git a/src/ElasticsearchProvider.php b/src/ElasticsearchProvider.php index 6b6d696..c95d979 100644 --- a/src/ElasticsearchProvider.php +++ b/src/ElasticsearchProvider.php @@ -5,6 +5,8 @@ use Laravel\Scout\EngineManager; use Illuminate\Support\ServiceProvider; use Elasticsearch\ClientBuilder as ElasticBuilder; +use ScoutEngines\Elasticsearch\Console\ElasticIndicesCommand; +use ScoutEngines\Elasticsearch\Console\ElasticMakeIndicesCommand; class ElasticsearchProvider extends ServiceProvider { @@ -15,10 +17,24 @@ public function boot() { resolve(EngineManager::class)->extend('elasticsearch', function($app) { return new ElasticsearchEngine(ElasticBuilder::create() - ->setHosts(config('scout.elasticsearch.hosts')) + ->setHosts(config('elasticsearch.hosts')) ->build(), - config('scout.elasticsearch.index') + config('elasticsearch.queries') ); }); } + + public function register() + { + if ($this->app->runningInConsole()) { + $this->commands([ + ElasticIndicesCommand::class, + ElasticMakeIndicesCommand::class + ]); + + $this->publishes([ + __DIR__ . '/../config/elasticsearch.php' => config_path('elasticsearch.php'), + ]); + } + } } diff --git a/tests/ElasticsearchEngineTest.php b/tests/ElasticsearchEngineTest.php index c292e01..1deb8d3 100644 --- a/tests/ElasticsearchEngineTest.php +++ b/tests/ElasticsearchEngineTest.php @@ -5,6 +5,14 @@ class ElasticsearchEngineTest extends PHPUnit_Framework_TestCase { + public $queryConfig = [ + 'default' => 'query_string', + + 'query_string' => [ + 'default_operator' => "AND" + ] + ]; + public function tearDown() { Mockery::close(); @@ -29,7 +37,7 @@ public function test_update_adds_objects_to_index() ] ]); - $engine = new ElasticsearchEngine($client, 'scout'); + $engine = new ElasticsearchEngine($client, $this->queryConfig); $engine->update(Collection::make([new ElasticsearchEngineTestModel])); } @@ -48,7 +56,7 @@ public function test_delete_removes_objects_to_index() ] ]); - $engine = new ElasticsearchEngine($client, 'scout'); + $engine = new ElasticsearchEngine($client, $this->queryConfig); $engine->delete(Collection::make([new ElasticsearchEngineTestModel])); } @@ -62,24 +70,45 @@ public function test_search_sends_correct_parameters_to_elasticsearch() 'query' => [ 'bool' => [ 'must' => [ - ['query_string' => ['query' => '*zonda*']], - ['match_phrase' => ['foo' => 1]] + [ + 'query_string' => [ + 'query' => 'zonda', + 'default_operator' => "AND" + ] + ] + ], + 'filter' => [ + [ + 'term' => [ + 'foo' => 1 + ] + ] ] ] - ] + ], + 'sort' => [ + '_score', + [ + 'title' => [ + 'order' => 'desc' + ] + ] + ], + 'track_scores' => true, ] ]); - $engine = new ElasticsearchEngine($client, 'scout'); + $engine = new ElasticsearchEngine($client, $this->queryConfig); $builder = new Laravel\Scout\Builder(new ElasticsearchEngineTestModel, 'zonda'); $builder->where('foo', 1); + $builder->orderBy('title', 'desc'); $engine->search($builder); } public function test_map_correctly_maps_results_to_models() { $client = Mockery::mock('Elasticsearch\Client'); - $engine = new ElasticsearchEngine($client, 'scout'); + $engine = new ElasticsearchEngine($client, $this->queryConfig); $model = Mockery::mock('Illuminate\Database\Eloquent\Model'); $model->shouldReceive('getKeyName')->andReturn('id'); @@ -108,6 +137,11 @@ public function getIdAttribute() return 1; } + public function searchableWithin() + { + return 'scout'; + } + public function searchableAs() { return 'table';