Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
No results found
Show changes
Commits on Source (32)
Showing
with 716 additions and 23 deletions
IQ_ES_HOSTS=127.0.0.1:9200
IQ_ES_USER=
IQ_ES_PASSWORD=
IQ_ES_PRODUCT_SEARCH_INDEX=
\ No newline at end of file
.DS_Store
/composer.lock
/vendor/
/.idea/
\ No newline at end of file
/.idea/
.env
.phpunit.result.cache
\ No newline at end of file
......@@ -9,7 +9,6 @@
"email": "p.piligrimov@iqdev.digital"
}
],
"version": "0.0.1",
"type": "library",
"keywords": [
"search",
......@@ -17,9 +16,8 @@
"php"
],
"require": {
"php": ">=7.4",
"ramsey/collection": "^1.2",
"iqdev/search-dc": "dev-main",
"php": ">=8.1",
"ramsey/collection": "^2",
"elasticsearch/elasticsearch": "^8.5",
"vlucas/phpdotenv": "^5.4.1"
},
......@@ -28,10 +26,21 @@
"IQDEV\\ElasticSearch\\": "src/ElasticSearch/"
}
},
"repositories": [
{
"type": "vcs",
"url": "ssh://git@gitlab.iqdev.digital:8422/piligrimov/search-dc.git"
"autoload-dev": {
"psr-4": {
"IQDEV\\ElasticSearchTests\\": "tests/"
}
},
"require-dev": {
"phpunit/phpunit": "^9.5",
"symfony/var-dumper": "^6.3"
},
"scripts": {
"tests": "php ./vendor/bin/phpunit --testdox --verbose"
},
"config": {
"allow-plugins": {
"php-http/discovery": true
}
]
}
}
<?xml version="1.0" encoding="UTF-8"?>
<phpunit colors="true"
convertErrorsToExceptions="true"
convertNoticesToExceptions="true"
convertWarningsToExceptions="true"
processIsolation="false"
stopOnFailure="false"
bootstrap="tests/bootstrap.php">
<testsuites>
<testsuite name="Project Tests Suite">
<directory>tests</directory>
</testsuite>
</testsuites>
<coverage processUncoveredFiles="true">
<include>
<directory suffix=".php">src/ElasticSearch/</directory>
</include>
</coverage>
</phpunit>
\ No newline at end of file
# Проведение тестов
Для работы корректности фильтрации и индексации данных в эластике необходимо подключение к elasticsearch
## Индексация elasticsearch
Эластик поднимается и настраивается отдельно.
Настроить подключение к эластику в переменных окружения .env
```dotenv
IQ_ES_HOSTS=http://127.0.0.1:9200
IQ_ES_USER=elastic
IQ_ES_PASSWORD=passsword
IQ_ES_PRODUCT_SEARCH_INDEX=product-test
```
Для наполнения эластика данными выполнить команду:
`php tests/CLI/DefaultSeed.php`
## Запуск тестов
Для проведения тестов можно использовать команду
`php composer tests`
\ No newline at end of file
<?php
namespace Docke\ElasticSearch\Config;
namespace IQDEV\ElasticSearch\Config;
use IQDEV\ElasticSearch\Configuration;
......@@ -15,4 +15,9 @@ class BaseConfiguration implements Configuration
{
return include __DIR__.'/product.mappings.php';
}
}
\ No newline at end of file
public function getSettings(): array
{
return include __DIR__.'/product.settings.php';
}
}
<?php
namespace IQDEV\ElasticSearch\Config;
use IQDEV\ElasticSearch\Configuration;
class MappingValidator
{
/**
* Проверка существовая поля в описании индекса
*
* @param Configuration $configuration
* @param string $property
*
* @return bool
*/
public static function isPropertyExists(Configuration $configuration, string $property): bool
{
$properties = array_keys($configuration->getMapping()['properties'] ?? []);
return in_array($property, $properties, true);
}
}
......@@ -9,8 +9,25 @@ return [
'full_search_content' => [
'type' => 'text',
],
'suggest_search_content' => [
'type' => 'text',
'analyzer' => 'autocomplete',
'search_analyzer' => 'standard',
],
'category_id' => [
'type' => 'keyword',
'type' => 'keyword',
'index' => false,
],
'rating' => [
'type' => 'double',
'index' => true,
],
'new' => [
'type' => 'boolean',
'index' => true
],
'popular' => [
'type' => 'double',
'index' => false,
],
'search_data' => [
......@@ -20,28 +37,28 @@ return [
'type' => 'nested',
'properties' => [
'facet_code' => [
'type' => 'keyword',
'index' => true
'type' => 'keyword',
'index' => true
],
'facet_value' => [
'type' => 'keyword',
'index' => true
],
'type' => 'keyword',
'index' => true
]
]
],
'number_facet' => [
'type' => 'nested',
'properties' => [
'facet_code' => [
'type' => 'keyword',
'index' => true
'type' => 'keyword',
'index' => true
],
'facet_value' => [
'type' => 'double'
'type' => 'double'
]
]
]
]
]
],
];
\ No newline at end of file
];
<?php
return [
'index' => [
'analysis' => [
'char_filter' => [
'eCharFilter' => [
'type' => 'mapping',
'mappings' => [
'Ё=>Е',
'ё=>е',
',=>.',
]
]
],
'analyzer' => [
'search_analyzer' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => [
'lowercase',
'russian_stemmer',
'russian_stop',
'synonym_filter',
'shingle',
],
'char_filter' => ['eCharFilter']
],
'autocomplete' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => [
'lowercase',
'autocomplete_filter'
]
]
],
'filter' => [
'shingle' => [
'type' => 'shingle',
'min_shingle_size' => 2,
'max_shingle_size' => 3
],
'russian_stop' => [
'type' => 'stop',
],
'russian_stemmer' => [
'type' => 'stemmer',
'language' => 'russian'
],
'synonym_filter' => [
'type' => 'synonym_graph',
'expand' => false,
'synonyms' => []
],
'autocomplete_filter' => [
'type' => 'edge_ngram',
'min_gram' => 1,
'max_gram' => 10,
]
]
],
]
];
\ No newline at end of file
......@@ -7,4 +7,6 @@ interface Configuration
public function getIndexName(): string;
public function getMapping(): array;
}
\ No newline at end of file
public function getSettings(): array;
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Aggregation;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Criteria\Criteria;
use IQDEV\ElasticSearch\Criteria\Filter\FilterType;
use IQDEV\ElasticSearch\Search\Aggs\AggsCollection;
use IQDEV\ElasticSearch\Search\Aggs\AggsFacetStats;
use IQDEV\ElasticSearch\Search\Aggs\AggsFacetTerms;
class Aggregation
{
public function __construct(
private readonly Configuration $configuration,
private readonly Criteria $criteria,
private readonly AggsCollection $aggregations = new AggsCollection(),
) {
$this->convertToQuery();
}
public function convertToQuery(): void
{
$queryFilterCollection = $this->criteria->getFilters()->getFilterCollectionByType(FilterType::QUERY);
if (false === $this->criteria->getSearch()->isEmpty() || false === $this->criteria->getFilters()->isEmpty()) {
$this->aggregations->add(
AggsFacetTerms::create(
'keyword_facet', 'keyword_facet'
)
);
$this->aggregations->add(
AggsFacetStats::create(
'number_facet', 'number_facet'
)
);
$filterAggregation = new FilterAggregation($this->configuration, $queryFilterCollection);
$filterAggregation->updateRequestAggregation($this->aggregations);
$fullAggregation = new FullAggregation($this->configuration, $queryFilterCollection);
$fullAggregation->updateRequestAggregation($this->aggregations);
}
if (false === $this->criteria->getAggs()->isEmpty()) {
$propertyAggregation = new PropertyAggregation(
$this->configuration,
$queryFilterCollection,
$this->criteria->getAggs(),
);
$propertyAggregation->updateRequestAggregation($this->aggregations);
}
}
public function getAggregation(): AggsCollection
{
return $this->aggregations;
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Aggregation;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Converter\Request\FilterQuery;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterCollection;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterGroupCollection;
use IQDEV\ElasticSearch\Criteria\Filter\Filter;
use IQDEV\ElasticSearch\Search\Aggs\Aggs;
use IQDEV\ElasticSearch\Search\Aggs\AggsCollection;
use IQDEV\ElasticSearch\Search\Aggs\AggsFacetTerms;
use IQDEV\ElasticSearch\Search\Nested;
class FilterAggregation
{
private ?Aggs $aggregation = null;
public function __construct(
private readonly Configuration $configuration,
private readonly FilterCollection $filterCollection,
) {
}
public function updateRequestAggregation(AggsCollection $original): void
{
foreach ($this->filterCollection as $filterGroup) {
/** @var FilterGroupCollection $filterGroup */
foreach ($filterGroup as $filter) {
/** @var Filter $filter */
$field = $filter->field()->value();
$aggregation = new Aggs($this->getKey('keyword', $field));
$aggregation->addAggs(
AggsFacetTerms::create(
'agg_special',
'keyword_facet'
)
);
$queryFilterBuilder = new FilterQuery($this->configuration, $this->filterCollection, [$field]);
$query = $queryFilterBuilder->getQuery();
if (false === $query->isEmpty()) {
$aggregation->setQuery($query);
} else {
$aggregation->setNested((new Nested())->setPath('search_data'));
}
$original->add($aggregation);
}
}
}
private function getKey(string $type, string $name): string
{
return sprintf('%s_facet_%s', $type, $name);
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Aggregation;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Converter\Request\FilterQuery;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterCollection;
use IQDEV\ElasticSearch\Search\Aggs\Aggs;
use IQDEV\ElasticSearch\Search\Aggs\AggsCollection;
use IQDEV\ElasticSearch\Search\Aggs\AggsFacetStats;
use IQDEV\ElasticSearch\Search\Aggs\AggsFacetTerms;
use IQDEV\ElasticSearch\Search\Nested;
class FullAggregation
{
public function __construct(
private readonly Configuration $configuration,
private readonly FilterCollection $filterCollection,
) {
}
public function updateRequestAggregation(AggsCollection $original): void
{
$keywordQuery = (new FilterQuery($this->configuration, $this->filterCollection->getKeywordFilters()))->getQuery();
$numberQuery = (new FilterQuery($this->configuration, $this->filterCollection->getNumberFilters()))->getQuery();
$aggsKeywordFiltered = new Aggs('keyword_facet_filtered');
$aggsKeywordFiltered->addAggs(
AggsFacetTerms::create(
'all_keyword_facet_filtered',
'keyword_facet'
)
);
$aggsNumberFiltered = new Aggs('number_facet_filtered');
$aggsNumberFiltered->addAggs(
AggsFacetStats::create(
'all_number_facet_filtered',
'number_facet'
)
);
if (false === $keywordQuery->getFilter()->isEmpty()) {
foreach ($keywordQuery->getFilter() as $item) {
$numberQuery->getFilter()->add($item);
}
}
if (false === $numberQuery->getFilter()->isEmpty()) {
foreach ($numberQuery->getFilter() as $item) {
$keywordQuery->getFilter()->add($item);
}
}
if (false === $keywordQuery->getFilter()->isEmpty()) {
$aggsKeywordFiltered->setQuery($keywordQuery);
} else {
$aggsKeywordFiltered->setNested((new Nested())->setPath('search_data'));
}
if (false === $numberQuery->getFilter()->isEmpty()) {
$aggsNumberFiltered->setQuery($numberQuery);
} else {
$aggsNumberFiltered->setNested((new Nested())->setPath('search_data'));
}
$original->add($aggsKeywordFiltered)->add($aggsNumberFiltered);
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Aggregation;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Converter\Request\FilterQuery;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterCollection;
use IQDEV\ElasticSearch\Document\Property\PropertyType;
use IQDEV\ElasticSearch\Search\Aggs\Aggs;
use IQDEV\ElasticSearch\Search\Aggs\AggsCollection;
use IQDEV\ElasticSearch\Search\Aggs\Terms;
class PropertyAggregation
{
public function __construct(
private readonly Configuration $configuration,
private readonly FilterCollection $filterCollection,
private readonly \IQDEV\ElasticSearch\Criteria\Aggs\AggsCollection $aggsCollection,
) {
}
public function updateRequestAggregation(AggsCollection $original): void
{
$queryFilterBuilder = new FilterQuery($this->configuration, $this->filterCollection);
$query = $queryFilterBuilder->getQuery();
/** @var \IQDEV\ElasticSearch\Criteria\Aggs\Aggs $aggs */
foreach ($this->aggsCollection as $aggs) {
$property = $aggs->getProperty();
if ($property->getType() !== PropertyType::BASE) {
continue;
}
$aggs = new Aggs($property->getKey());
if (false === $query->isEmpty()) {
$aggs->setQuery($query);
$aggs->addAggs(
(new Aggs($property->getKey()))
->setTerms(new Terms($property->getKey()))
);
} else {
$aggs->setTerms(new Terms($property->getKey()));
}
$aggs->addAggs(
(new Aggs($property->getKey()))
->setTerms(new Terms($property->getKey()))
);
$original->add($aggs);
}
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Collection;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterCollection;
class NestedFilterCollection extends FilterCollection
{
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Collection;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterCollection;
class PropertyFilterCollection extends FilterCollection
{
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Converter\Request\Aggregation\Aggregation;
use IQDEV\ElasticSearch\Criteria\Criteria;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\PostFilterCollection;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\QueryFilterCollection;
use IQDEV\ElasticSearch\Criteria\Search\Search;
use IQDEV\ElasticSearch\Criteria\Search\SearchQuery;
use IQDEV\ElasticSearch\Document\Property\PropertyType;
use IQDEV\ElasticSearch\Search\Pagination;
class CriteriaRequestBuilder extends RequestBuilder
{
public function __construct(
private readonly Configuration $configuration,
private readonly Criteria $criteria
) {
parent::__construct();
}
public function build(): void
{
$this->setPagination();
if (false === $this->criteria->getSorting()->isEmpty()) {
$this->setSort();
}
if (false === $this->criteria->getSearch()->isEmpty()) {
$this->setSearch();
}
if (false === $this->criteria->getFilters()->isEmpty()) {
$this->setFilter();
}
$this->setAggregations();
}
public function setPagination(): void
{
$request = clone $this->request;
$request->setPagination(new Pagination(
$this->criteria->getPagination()->limit,
$this->criteria->getPagination()->offset
));
$this->request = $request;
}
public function setSort(): void
{
$request = clone $this->request;
foreach ($this->criteria->getSorting() as $sort) {
$request->getSort()->add($sort);
}
$this->request = $request;
}
public function setSearch(): void
{
$request = clone $this->request;
foreach ($this->criteria->getSearch() as $search) {
/** @var Search $search */
$searchQuery = new SearchQuery($search);
if ($search->getProperty()->getType() === PropertyType::TEXT) {
$request->getQueryMatch()->add($searchQuery->toQueryMatch());
} else {
$request->getQuery()->getMust()->add($searchQuery->toMust($this->configuration));
}
}
$this->request = $request;
}
public function setFilter(): void
{
$queryFilterCollection = $this->criteria->getFilters()->getQueryFilterCollection();
if (false === $queryFilterCollection->isEmpty()) {
$this->setQueryFilter($queryFilterCollection);
}
$postFilterCollection = $this->criteria->getFilters()->getPostFilterCollection();
if (false === $postFilterCollection->isEmpty()) {
$this->setPostFilter($postFilterCollection);
}
}
private function setQueryFilter(QueryFilterCollection $filterCollection): void
{
$request = clone $this->request;
$filterQuery = new FilterQuery($this->configuration, $filterCollection);
$request->getQuery()->modify($filterQuery->getQuery());
$this->request = $request;
}
private function setPostFilter(PostFilterCollection $filterCollection): void
{
$request = clone $this->request;
$filterQuery = new FilterQuery($this->configuration, $filterCollection);
$request->getPostFilter()->modify($filterQuery->getQuery());
$this->request = $request;
}
public function setAggregations(): void
{
$request = clone $this->request;
$aggregation = new Aggregation($this->configuration, $this->criteria);
$request->setAggs($aggregation->getAggregation());
$this->request = $request;
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request;
use IQDEV\ElasticSearch\Configuration;
use IQDEV\ElasticSearch\Criteria\Criteria;
use IQDEV\ElasticSearch\Search\Request;
final class CriteriaToRequest
{
public function __construct(
private readonly Configuration $configuration,
) {
}
public function fromCriteria(Criteria $criteria): Request
{
$builder = new CriteriaRequestBuilder($this->configuration, $criteria);
$builder->build();
return $builder->getRequest();
}
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Filter;
use IQDEV\ElasticSearch\Esable;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterGroupCollection;
use IQDEV\ElasticSearch\Criteria\Filter\LogicOperator;
use IQDEV\ElasticSearch\Search\BoolQuery\Query;
use IQDEV\ElasticSearch\Search\Nested;
abstract class AbstractFilterQuery
{
protected Query $query;
public function __construct() {
$this->query = new Query();
}
protected function setFilterByLogic(Query $query, LogicOperator $logicOperator, Esable $filter): void
{
match ($logicOperator) {
LogicOperator::AND => $query->getFilter()->add($filter),
LogicOperator::OR => $query->getShould()->add($filter),
LogicOperator::NOT => $query->getMustNot()->add($filter),
};
}
abstract public function getQuery(FilterGroupCollection $filterGroupCollection, array $exclude): Query|Nested;
}
<?php
declare(strict_types=1);
namespace IQDEV\ElasticSearch\Converter\Request\Filter;
use IQDEV\ElasticSearch\Converter\Request\Filter\Type\KeywordFilterType;
use IQDEV\ElasticSearch\Converter\Request\Filter\Type\RangeFilterType;
use IQDEV\ElasticSearch\Criteria\Filter\Collection\FilterGroupCollection;
use IQDEV\ElasticSearch\Criteria\Filter\Filter;
use IQDEV\ElasticSearch\Search\BoolQuery\Query;
use IQDEV\ElasticSearch\Search\BoolQuery\Terms;
use IQDEV\ElasticSearch\Search\Nested;
class NestedFilter extends AbstractFilterQuery
{
public const NESTED_RANGE_PATH = 'search_data.number_facet';
public const NESTED_KEYWORD_PATH = 'search_data.keyword_facet';
public function getQuery(FilterGroupCollection $filterGroupCollection, array $exclude): Nested
{
$query = new Query();
$keywordFiltersGroup = $filterGroupCollection->getKeywordFilters($exclude);
foreach ($keywordFiltersGroup as $keywordFilter) {
/** @var Filter $keywordFilter */
$esableFilter = new KeywordFilterType($keywordFilter);
$this->setFilterByLogic($query, $filterGroupCollection->getLogicOperator(), $this->getNested($esableFilter, self::NESTED_KEYWORD_PATH));
}
$rangeFilterGroup = $filterGroupCollection->getRangeFilters($exclude);
$rangesFilter = RangeFilterType::getFiltersByOneProperty($rangeFilterGroup);
foreach ($rangesFilter as $filterGroup) {
/** @var FilterGroupCollection $filterGroup */
$esableFilter = new RangeFilterType($filterGroup);
$this->setFilterByLogic($query, $filterGroupCollection->getLogicOperator(), $this->getNested($esableFilter, self::NESTED_RANGE_PATH));
}
return (new Nested())
->setPath('search_data')
->setQuery($query);
}
private function getNested(RangeFilterType|KeywordFilterType $filter, string $path): Nested
{
$cloneFilter = clone $filter;
$nested = new Nested();
$query = new Query();
$query->getFilter()->add(new Terms($path . '.facet_code', $cloneFilter->getField()));
$cloneFilter->setField($path . '.facet_value');
$query->getFilter()->add($cloneFilter->getEsable());
$nested
->setPath($path)
->setQuery($query);
return $nested;
}
}