diff --git a/README.md b/README.md index 425dc12..c5d7b16 100644 --- a/README.md +++ b/README.md @@ -144,17 +144,18 @@ and setting the `trgm_feature_query`, `trgm_layer_query`, `trgm_similarity_thres Config options in the config file can be overridden by equivalent uppercase environment variables. -| Variable | Description | Default value | -|---------------------|------------------------------------------|-----------------------------------------| -| SEARCH_BACKEND | Search backend | `solr` | -| SOLR_SERVICE_URL | SOLR service URL | `http://localhost:8983/solr/gdi/select` | -| WORD_SPLIT_RE | Word split Regex | `[\s,.:;"]+` | -| SEARCH_RESULT_LIMIT | Result count limit per search | `50` | -| SEARCH_RESULT_SORT | Sorting of search results (solr backend) | `score desc, sort asc` | -| DB_URL | DB connection for search geometries view | | -| TRGM_FEATURE_QUERY | Feature query SQL (trigram backend) | | -| TRGM_LAYER_QUERY | Layer query SQL (trigram backend) | | -| TRGM_SIMILARITY_THRESHOLD | Trigram similarity treshold (trigram backend) | `0.3` | +| Variable | Description | Default value | +|-----------------------------|------------------------------------------|-----------------------------------------| +| SEARCH_BACKEND | Search backend | `solr` | +| SOLR_SERVICE_URL | SOLR service URL | `http://localhost:8983/solr/gdi/select` | +| WORD_SPLIT_RE | Word split Regex | `[\s,.:;"]+` | +| SEARCH_RESULT_LIMIT | Result count limit per search | `50` | +| SEARCH_RESULT_SORT | Sorting of search results (solr backend) | `score desc, sort asc` | +| DB_URL | DB connection for search geometries view | | +| TRGM_FEATURE_QUERY | Feature query SQL (trigram backend) | | +| TRGM_FEATURE_QUERY_TEMPLATE | Feature query SQL Jinja template (trigram backend) | | +| TRGM_LAYER_QUERY_TEMPLATE | Layer query SQL Jinja template (trigram backend) | | +| TRGM_SIMILARITY_THRESHOLD | Trigram similarity treshold (trigram backend) | `0.3` | Usage/Development diff --git a/requirements.txt b/requirements.txt index 6227641..d417610 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ requests==2.32.0 psycopg2==2.9.9 SQLAlchemy==2.0.29 qwc-services-core==1.3.34 +Jinja2==3.1.4 diff --git a/schemas/qwc-search-service.json b/schemas/qwc-search-service.json index 3d44586..228fafc 100644 --- a/schemas/qwc-search-service.json +++ b/schemas/qwc-search-service.json @@ -32,10 +32,18 @@ "description": "Search result ordering for solr search results. Default: search_result_sort", "type": "string" }, + "trgm_feature_query_template": { + "description": "Jinja template string to generate the feature query SQL. The following variables are passed to the template: `searchtext` (the full search text, as a string), `words` (the single words of the search text, as an array), `facets` (the permitted search facets, as an array). The generated SQL needs to satisfy the same requirements as documented in `trgm_feature_query`.", + "type": "string" + }, "trgm_feature_query": { "description": "Trigram feature query SQL. You can use the placeholder parameters `:term` (full search string), `:terms` (list of words of the search string) and `:thres` (similarity threshold). The query must return the columns display, facet_id, id_field_name, feature_id, bbox (as a `[xmin,ymin,xmax,ymax]` string), srid.", "type": "string" }, + "trgm_layer_query_template": { + "description": "Jinja template string to generate the layer query SQL. The following variables are passed to the template: `searchtext` (the full search text, as a string), `words` (the single words of the search text, as an array), `facets` (the permitted search facets, as an array). The generated SQL needs to satisfy the same requirements as documented in `trgm_layer_query`.", + "type": "string" + }, "trgm_layer_query": { "description": "Trigram layer query SQL. You can use the placeholder parameters `:term` (full search string), `:terms` (list of words of the search string) and `:thres` (similarity threshold). The query must return the columns display, dataproduct_id, dset_info, sublayers.", "type": "string" diff --git a/src/trgm_search_service.py b/src/trgm_search_service.py index 5f7345c..4ae3c17 100644 --- a/src/trgm_search_service.py +++ b/src/trgm_search_service.py @@ -7,6 +7,7 @@ from qwc_services_core.permissions_reader import PermissionsReader from qwc_services_core.runtime_config import RuntimeConfig from flask import json, request +from jinja2 import Template from sqlalchemy.sql import text as sql_text @@ -47,7 +48,9 @@ def __init__(self, tenant, logger): self.db_url = config.get('db_url') self.filter_word_query = config.get('trgm_filter_word_query') self.feature_query = config.get('trgm_feature_query') + self.feature_query_template = config.get('trgm_feature_query_template') self.layer_query = config.get('trgm_layer_query') + self.layer_query_template = config.get('trgm_layer_query_template') self.similarity_threshold = config.get('trgm_similarity_threshold', 0.3) def search(self, identity, searchtext, filter, limit): @@ -71,6 +74,17 @@ def search(self, identity, searchtext, filter, limit): if not limit: limit = self.default_search_limit + # Prepare query + layer_query = self.layer_query + if self.layer_query_template: + layer_query = Template(self.layer_query_template).render(searchtext=searchtext, words=tokens, facets=search_permissions) + self.logger.debug("Generated layer query from template") + + feature_query = self.feature_query + if self.feature_query_template: + feature_query = Template(self.feature_query_template).render(searchtext=searchtext, words=tokens, facets=search_permissions) + self.logger.debug("Generated feature query from template") + # Perform search layer_results = [] feature_results = [] @@ -80,17 +94,17 @@ def search(self, identity, searchtext, filter, limit): conn.execute(sql_text("SET pg_trgm.similarity_threshold = :value"), {'value': self.similarity_threshold}) # Search for layers - if self.layer_query and search_dataproducts: + if layer_query and search_dataproducts: start = time.time() - self.logger.debug("Searching for layers: %s" % self.layer_query) - layer_results = conn.execute(sql_text(self.layer_query), {'term': " ".join(tokens), 'terms': tokens, 'thres': self.similarity_threshold}).mappings().all() + self.logger.debug("Searching for layers: %s" % layer_query) + layer_results = conn.execute(sql_text(layer_query), {'term': " ".join(tokens), 'terms': tokens, 'thres': self.similarity_threshold}).mappings().all() self.logger.debug("Done in %f s" % (time.time() - start)) # Search for features - if self.feature_query: + if feature_query: start = time.time() - self.logger.debug("Searching for features: %s" % self.feature_query) - feature_results = conn.execute(sql_text(self.feature_query), {'term': " ".join(tokens), 'terms': tokens, 'thres': self.similarity_threshold}).mappings().all() + self.logger.debug("Searching for features: %s" % feature_query) + feature_results = conn.execute(sql_text(feature_query), {'term': " ".join(tokens), 'terms': tokens, 'thres': self.similarity_threshold}).mappings().all() self.logger.debug("Done in %f s" % (time.time() - start)) # Build results