From ac3e9dd05342c79f408e4fd76710dc41584be2cf Mon Sep 17 00:00:00 2001 From: Vadim Galizyanov Date: Wed, 3 May 2023 15:49:31 +0500 Subject: [PATCH] separation of filters into post and query --- .env.example | 4 + .gitignore | 4 +- composer.json | 12 + phpunit.xml.dist | 24 ++ .../Converter/CriteriaToEsRequest.php | 92 +++-- .../Filter/PostFilterCollection.php | 9 + .../Filter/QueryFilterCollection.php | 9 + tests/AbstractTestCase.php | 21 ++ tests/FIlter/QueryTest.php | 336 ++++++++++++++++++ tests/Factory/ClientFactory.php | 25 ++ tests/Helpers/Arr.php | 24 ++ tests/Seed/DefaultSeed.php | 138 +++++++ tests/Service/SearchClient.php | 29 ++ tests/bootstrap.php | 8 + 14 files changed, 714 insertions(+), 21 deletions(-) create mode 100644 .env.example create mode 100644 phpunit.xml.dist create mode 100644 src/ElasticSearch/Filter/PostFilterCollection.php create mode 100644 src/ElasticSearch/Filter/QueryFilterCollection.php create mode 100644 tests/AbstractTestCase.php create mode 100644 tests/FIlter/QueryTest.php create mode 100644 tests/Factory/ClientFactory.php create mode 100644 tests/Helpers/Arr.php create mode 100644 tests/Seed/DefaultSeed.php create mode 100644 tests/Service/SearchClient.php create mode 100644 tests/bootstrap.php diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..901dac0 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +IQ_ES_HOSTS=127.0.0.1:9200 +IQ_ES_USER= +IQ_ES_PASSWORD= +IQ_ES_PRODUCT_SEARCH_INDEX= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4418083..5249d08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .DS_Store /composer.lock /vendor/ -/.idea/ \ No newline at end of file +/.idea/ +.env +.phpunit.result.cache \ No newline at end of file diff --git a/composer.json b/composer.json index e95c6a6..6d930cb 100644 --- a/composer.json +++ b/composer.json @@ -27,12 +27,24 @@ "IQDEV\\ElasticSearch\\": "src/ElasticSearch/" } }, + "autoload-dev": { + "psr-4": { + "IQDEV\\ElasticSearchTests\\": "tests/" + } + }, "repositories": [ { "type": "vcs", "url": "ssh://git@gitlab.iqdev.digital:8422/piligrimov/search-dc.git" } ], + "require-dev": { + "phpunit/phpunit": "^9.5", + "symfony/var-dumper": "^5.4" + }, + "scripts": { + "tests": "php ./vendor/bin/phpunit --testdox --verbose" + }, "config": { "allow-plugins": { "php-http/discovery": true diff --git a/phpunit.xml.dist b/phpunit.xml.dist new file mode 100644 index 0000000..9c3b361 --- /dev/null +++ b/phpunit.xml.dist @@ -0,0 +1,24 @@ + + + + + + tests + + + + + + src/ElasticSearch/ + + + + + + \ No newline at end of file diff --git a/src/ElasticSearch/Converter/CriteriaToEsRequest.php b/src/ElasticSearch/Converter/CriteriaToEsRequest.php index b5b50c6..d0ea8a6 100644 --- a/src/ElasticSearch/Converter/CriteriaToEsRequest.php +++ b/src/ElasticSearch/Converter/CriteriaToEsRequest.php @@ -2,6 +2,8 @@ namespace IQDEV\ElasticSearch\Converter; +use IQDEV\ElasticSearch\Filter\PostFilterCollection; +use IQDEV\ElasticSearch\Filter\QueryFilterCollection; use IQDEV\ElasticSearch\Order\OrderAscType; use IQDEV\ElasticSearch\Order\OrderDescType; use IQDEV\ElasticSearch\Order\OrderField; @@ -24,6 +26,7 @@ use IQDEV\Search\FIlter\Filter; use IQDEV\Search\Filter\FilterCollection; use IQDEV\Search\Filter\FilterGroupCollection; use IQDEV\Search\Filter\FilterOperator; +use IQDEV\Search\Filter\FilterType; use IQDEV\Search\Filter\LogicOperator; use IQDEV\Search\Order\Order; use IQDEV\Search\Order\OrderAscType as SOrderAscType; @@ -126,34 +129,56 @@ final class CriteriaToEsRequest } } - $keywordFilter = $this->getKeywordFilter($criteria); - if (false === $keywordFilter->isEmpty()) { + [$queryFilters, $postFilters] = $this->groupFilters($criteria->filters()); + + $keywordQueryFilter = $this->getKeywordFilter($queryFilters); + $keywordPostFilter = $this->getKeywordFilter($postFilters); + if (false === $keywordQueryFilter->isEmpty() || false === $keywordPostFilter->isEmpty()) { $keywordNestedFilter = new Nested(); - $keywordNestedFilter->setPath('search_data') - ->setQuery($keywordFilter); + $keywordNestedFilter->setPath('search_data'); + + if (false === $keywordQueryFilter->isEmpty()) { + $keywordNestedFilterQuery = clone $keywordNestedFilter; + $keywordNestedFilterQuery->setQuery($keywordQueryFilter); + $request->getQuery()->filter($keywordNestedFilterQuery); + } - $request->getPostFilter()->filter($keywordNestedFilter); + if (false === $keywordPostFilter->isEmpty()) { + $keywordNestedFilterPost = clone $keywordNestedFilter; + $keywordNestedFilterPost->setQuery($keywordPostFilter); + $request->getPostFilter()->filter($keywordNestedFilterPost); + } } - $numberFilter = $this->getNumberFilter($criteria); - if (false === $numberFilter->isEmpty()) { + $numberQueryFilter = $this->getNumberFilter($queryFilters); + $numberPostFilter = $this->getNumberFilter($postFilters); + if (false === $numberQueryFilter->isEmpty() || false === $numberPostFilter->isEmpty()) { $numberNestedFilter = new Nested(); - $numberNestedFilter->setPath('search_data') - ->setQuery($numberFilter); + $numberNestedFilter->setPath('search_data'); + + if (false === $numberQueryFilter->isEmpty()) { + $numberNestedFilterQuery = clone $numberNestedFilter; + $numberNestedFilterQuery->setQuery($numberQueryFilter); + $request->getQuery()->filter($numberNestedFilterQuery); + } - $request->getPostFilter()->filter($numberNestedFilter); + if (false === $numberPostFilter->isEmpty()) { + $numberNestedFilterPost = clone $numberNestedFilter; + $numberNestedFilterPost->setQuery($numberPostFilter); + $request->getPostFilter()->filter($numberNestedFilterPost); + } } return $request; } - private function getNumberFilter(Criteria $criteria, array $excludeFilter = []): Query + private function getNumberFilter(FilterCollection $filterCollection, array $excludeFilter = []): Query { $numberFilter = new Query(); $ranges = []; - foreach ($criteria->filters() as $filterGroup) { + foreach ($filterCollection as $filterGroup) { /** @var FilterGroupCollection $filterGroup */ if ($filterGroup->isEmpty()) { continue; @@ -201,10 +226,10 @@ final class CriteriaToEsRequest return $numberFilter; } - private function getKeywordFilter(Criteria $criteria, array $excludeFilter = []): Query + private function getKeywordFilter(FilterCollection $filterCollection, array $excludeFilter = []): Query { $keywordFilter = new Query(); - foreach ($criteria->filters() as $filterGroup) { + foreach ($filterCollection as $filterGroup) { /** @var FilterGroupCollection $filterGroup */ if ($filterGroup->isEmpty()) { continue; @@ -301,8 +326,8 @@ final class CriteriaToEsRequest ); $queryNumberFiltered = new Query(); - $keywordFilter = $this->getKeywordFilter($criteria); - $numberFilter = $this->getNumberFilter($criteria, [$field]); + $keywordFilter = $this->getKeywordFilter($criteria->filters()); + $numberFilter = $this->getNumberFilter($criteria->filters(), [$field]); if (false === $keywordFilter->isEmpty()) { $nestedFilterKeyword = new Nested(); @@ -348,8 +373,8 @@ final class CriteriaToEsRequest ) ); $queryKeywordFiltered = new Query(); - $keywordFilter = $this->getKeywordFilter($criteria, [$field]); - $numberFilter = $this->getNumberFilter($criteria); + $keywordFilter = $this->getKeywordFilter($criteria->filters(), [$field]); + $numberFilter = $this->getNumberFilter($criteria->filters()); if (false === $keywordFilter->isEmpty()) { $nestedFilterKeyword = new Nested(); $nestedFilterKeyword->setPath('search_data') @@ -373,8 +398,8 @@ final class CriteriaToEsRequest $request->getAggs()->add($aggsFiltered); } - $keywordFilter = $this->getKeywordFilter($criteria); - $numberFilter = $this->getNumberFilter($criteria); + $keywordFilter = $this->getKeywordFilter($criteria->filters()); + $numberFilter = $this->getNumberFilter($criteria->filters()); $aggsKeywordFiltered = new Aggs('keyword_facet_filtered'); $aggsKeywordFiltered->addAggs( @@ -431,4 +456,31 @@ final class CriteriaToEsRequest return $request; } + + /** + * @param FilterCollection $filters + * @return FilterCollection[] + */ + private function groupFilters(FilterCollection $filters): array + { + $queryFilters = new QueryFilterCollection(); + $postFilters = new PostFilterCollection(); + foreach ($filters as $filterGroup) { + /** @var FilterGroupCollection $filterGroup */ + if ($filterGroup->isEmpty()) { + continue; + } + + switch ($filterGroup->getFilterType()->value()) { + case FilterType::QUERY: + $queryFilters->add($filterGroup); + break; + case FilterType::POST: + $postFilters->add($filterGroup); + break; + } + } + + return [$queryFilters, $postFilters]; + } } diff --git a/src/ElasticSearch/Filter/PostFilterCollection.php b/src/ElasticSearch/Filter/PostFilterCollection.php new file mode 100644 index 0000000..315d43a --- /dev/null +++ b/src/ElasticSearch/Filter/PostFilterCollection.php @@ -0,0 +1,9 @@ + $value) { + $this->assertArrayHasKey($key, $actualKeys, $message); + if (isset($actualKeys[$key])) { + $this->assertEquals($actualKeys[$key], $value, $key); + } + } + } +} \ No newline at end of file diff --git a/tests/FIlter/QueryTest.php b/tests/FIlter/QueryTest.php new file mode 100644 index 0000000..6441afd --- /dev/null +++ b/tests/FIlter/QueryTest.php @@ -0,0 +1,336 @@ + [ + 'key' => 'category_id', + 'value' => 'shoes', + ], + 'brand' => [ + 'key' => 'brand', + 'value' => 'nike', + ], + 'price' => [ + 'key' => 'price', + 'max' => 100.0, + 'min' => 10.0, + ] + ]; + + $criteria = new Criteria(); + + + $filterCollectionCategory = new FilterGroupCollection([ + new Filter( + new Field($filter['category']['key']), + new FilterOperator(FilterOperator::EQ), + new FilterKeyword($filter['category']['value']) + ) + ]); + $filterCollectionCategory->setFilterType(FilterType::query()); + $criteria->filters()->add($filterCollectionCategory); + + $filterCollectionBrand = new FilterGroupCollection([ + new Filter( + new Field($filter['brand']['key']), + new FilterOperator(FilterOperator::EQ), + new FilterKeyword($filter['brand']['value']) + ) + ]); + + $filterCollectionPrice = new FilterGroupCollection([ + new Filter( + new Field($filter['price']['key']), + new FilterOperator(FilterOperator::LT), + new FilterNumber($filter['price']['min']) + ), + new Filter( + new Field($filter['price']['key']), + new FilterOperator(FilterOperator::GT), + new FilterNumber($filter['price']['max']) + ), + ]); + + // Формирование фильтра для post + $criteriaPost = clone $criteria; + $criteriaPost->filters()->add(clone $filterCollectionPrice); + $criteriaPost->filters()->add(clone $filterCollectionBrand); + + + // Формирование фильтра для query + $criteriaQuery = clone $criteria; + + $filterTypeQuery = FilterType::query(); + $filterCollectionPrice->setFilterType($filterTypeQuery); + $filterCollectionBrand->setFilterType($filterTypeQuery); + $criteriaQuery->filters()->add(clone $filterCollectionPrice); + $criteriaQuery->filters()->add(clone $filterCollectionBrand); + + + // Получение классов с данными для запроса в es + $criteriaToEsRequest = new CriteriaToEsRequest(); + $requestPost = $criteriaToEsRequest->fromCriteria($criteriaPost); + $requestQuery = $criteriaToEsRequest->fromCriteria($criteriaQuery); + + + $expectedFilter = [ + [ + "nested" => [ + "path" => "search_data", + "query" => [ + "bool" => [ + "filter" => [ + [ + "nested" => [ + "path" => "search_data.keyword_facet", + "query" => [ + "bool" => [ + "filter" => [ + [ + "term" => [ + "search_data.keyword_facet.facet_code" => $filter['brand']['key'] + ] + ], + [ + "term" => [ + "search_data.keyword_facet.facet_value" => $filter['brand']['value'] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ], + [ + "nested" => [ + "path" => "search_data", + "query" => [ + "bool" => [ + "filter" => [ + [ + "nested" => [ + "path" => "search_data.number_facet", + "query" => [ + "bool" => [ + "filter" => [ + [ + "term" => [ + "search_data.number_facet.facet_code" => $filter['price']['key'] + ] + ], + [ + "range" => [ + "search_data.number_facet.facet_value" => [ + "lt" => $filter['price']['min'], + "gt" => $filter['price']['max'], + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ]; + $expected = [ + "query" => [ + "bool" => [ + "must" => [ + [ + "term" => [ + "category_id" => $filter['category']['value'] + ] + ], + ] + ] + ], + ]; + + + $this->assertArray( + array_merge($expected, [ + "query" => [ + "bool" => [ + "filter" => $expectedFilter, + ] + ] + ]), + $requestQuery->es(), + 'query response' + ); + + + $this->assertArray( + array_merge($expected, [ + "post_filter" => [ + "bool" => [ + "filter" => $expectedFilter, + ] + ] + ]), + $requestPost->es(), + 'post response' + ); + } + + + public function testAddingASingleKeyFilterToPostAndQuery() + { + $filter = [ + 'price' => [ + 'key' => 'price', + 'max' => 100.0, + 'min' => 10.0, + 'lower' => 0.0, + ] + ]; + + $criteria = new Criteria(); + + $filterCollectionPrice = new FilterGroupCollection([ + new Filter( + new Field($filter['price']['key']), + new FilterOperator(FilterOperator::LT), + new FilterNumber($filter['price']['min']) + ), + new Filter( + new Field($filter['price']['key']), + new FilterOperator(FilterOperator::GT), + new FilterNumber($filter['price']['max']) + ), + ]); + $criteria->filters()->add($filterCollectionPrice); + + + $filterCollectionQueryPrice = new FilterGroupCollection([ + new Filter( + new Field($filter['price']['key']), + new FilterOperator(FilterOperator::LT), + new FilterNumber($filter['price']['lower']) + ), + ]); + $filterCollectionQueryPrice->setFilterType(FilterType::query()); + $criteria->filters()->add($filterCollectionQueryPrice); + + + $criteriaToEsRequest = new CriteriaToEsRequest(); + $request = $criteriaToEsRequest->fromCriteria($criteria); + + + $expected = [ + "post_filter" => [ + "bool" => [ + "filter" => [ + 0 => [ + "nested" => [ + "path" => "search_data", + "query" => [ + "bool" => [ + "filter" => [ + 0 => [ + "nested" => [ + "path" => "search_data.number_facet", + "query" => [ + "bool" => [ + "filter" => [ + [ + "term" => [ + "search_data.number_facet.facet_code" => "price" + ] + ], + [ + "range" => [ + "search_data.number_facet.facet_value" => [ + "lt" => $filter['price']['min'], + "gt" => $filter['price']['max'], + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ], + "query" => [ + "bool" => [ + "filter" => [ + 0 => [ + "nested" => [ + "path" => "search_data", + "query" => [ + "bool" => [ + "filter" => [ + 0 => [ + "nested" => [ + "path" => "search_data.number_facet", + "query" => [ + "bool" => [ + "filter" => [ + 0 => [ + "term" => [ + "search_data.number_facet.facet_code" => "price" + ] + ], + 1 => [ + "range" => [ + "search_data.number_facet.facet_value" => [ + "lt" => $filter['price']['lower'] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ]; + + + $this->assertArray($expected, $request->es()); + } +} \ No newline at end of file diff --git a/tests/Factory/ClientFactory.php b/tests/Factory/ClientFactory.php new file mode 100644 index 0000000..30f2480 --- /dev/null +++ b/tests/Factory/ClientFactory.php @@ -0,0 +1,25 @@ +setHosts(explode(',', $_ENV['IQ_ES_HOSTS'] ?: 'http://localhost:9200')) + ->setBasicAuthentication($_ENV['IQ_ES_USER'], $_ENV['IQ_ES_PASSWORD']) + ->build(); + + return self::$instance; + } +} \ No newline at end of file diff --git a/tests/Helpers/Arr.php b/tests/Helpers/Arr.php new file mode 100644 index 0000000..6164986 --- /dev/null +++ b/tests/Helpers/Arr.php @@ -0,0 +1,24 @@ + $item) { + if ($startKey !== null) { + $key = $startKey.'.'.$key; + } + + if (is_array($item)) { + $keys = array_merge($keys, static::convertingOneDimensional($item, $key)); + continue; + } + + $keys[$key] = $item; + } + return $keys; + } +} \ No newline at end of file diff --git a/tests/Seed/DefaultSeed.php b/tests/Seed/DefaultSeed.php new file mode 100644 index 0000000..f2390cd --- /dev/null +++ b/tests/Seed/DefaultSeed.php @@ -0,0 +1,138 @@ +configuration = new BaseConfiguration(); + + $this->indexRunner = new IndexRunner( + ClientFactory::create(), + $this->configuration, + new TestLogger() + ); + } + + public function start() + { + $provider = new class implements IndexProvider { + public function get(): \Generator + { + $products = [ + [ + 'id' => 's1', + 'name' => 'Кроссовки NMD_R1 Boba Fett Spectoo', + 'category' => 'shoes', + 'properties' => ['brand' => 'adidas', 'color' => 'green', 'size' => 46,'price' => 100] + ], + [ + 'id' => 's2', + 'name' => 'КРОССОВКИ ULTRABOOST 5.0 DNA', + 'category' => 'shoes', + 'properties' => ['brand' => 'adidas', 'color' => 'red', 'size' => 47,'price' => 101] + ], + [ + 'id' => 's3', + 'name' => 'Кроссовки Reebok Royal Techque', + 'category' => 'shoes', + 'properties' => ['brand' => 'rebook', 'color' => 'blue', 'size' => 47,'price' => 102] + ], + [ + 'id' => 's4', + 'name' => 'Nike Air Zoom Pegasus 39', + 'category' => 'shoes', + 'properties' => ['brand' => 'nike', 'color' => 'green', 'size' => 43,'price' => 103] + ], + [ + 'id' => 'h1', + 'name' => 'Nike Dri-FIT Strike', + 'category' => 't-short', + 'properties' => ['brand' => 'nike', 'color' => 'red', 'size' => 'xl','price' => 104] + ], + [ + 'id' => 'h2', + 'name' => 'Nike Dri-FIT Rise 365', + 'category' => 't-short', + 'properties' => ['brand' => 'nike', 'color' => 'white', 'size' => 'xxl','price' => 105] + ], + [ + 'id' => 'h3', + 'name' => 'Компрессионная Футболка ACTIVCHILL Graphic Move', + 'category' => 't-short', + 'properties' => ['brand' => 'rebook', 'color' => 'white', 'size' => 'xl','price' => 106] + ], + [ + 'id' => 'p1', + 'name' => 'Товар с ценой', + 'category' => 'prices', + 'properties' => ['brand' => 'rebook', 'color' => 'white', 'size' => 'xl','price' => 107] + ], + ]; + + foreach ($products as $product) { + $document = new ProductDocument(new FacetCategory($product['category'])); + //todo по-хорошему нужны базовые классы, которые будут описывать свойства + // и формировать структуру для последующей обработки + + $document->setAdditionData([ + 'id' => $product['id'], + 'title' => $product['name'], + ]); + + foreach ($product['properties'] as $key => $prop) { + if ($key === 'price') { + $document->getNumberFacets()->add(new FacetNumber($key, $prop)); + } else { + $document->getKeywordFacets()->add(new FacetKeyword($key, $prop)); + } + } + $document->setSearchContent($product['name']); + + yield new AddIndex( + $_ENV['IQ_ES_PRODUCT_SEARCH_INDEX'], + $document, + $product['id'] + ); + } + } + + public function setBatchSize(int $size): void + { + } + + public function getBatchSize(): ?int + { + return null; + } + + public function setLimit(int $limit): void + { + } + + public function getLimit(): ?int + { + return null; + } + }; + + $this->indexRunner->run($provider); + } +} \ No newline at end of file diff --git a/tests/Service/SearchClient.php b/tests/Service/SearchClient.php new file mode 100644 index 0000000..bb3d57a --- /dev/null +++ b/tests/Service/SearchClient.php @@ -0,0 +1,29 @@ +load(); -- GitLab