diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json index d5fb9697f89..e6f08f708a8 100644 --- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json +++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json @@ -47,7 +47,7 @@ "item_search_settings_section", "redisearch_warning", "search_index_fields", - "show_categories_in_search_autocomplete", + "is_redisearch_enabled", "is_redisearch_loaded", "shop_by_category_section", "slideshow", @@ -293,6 +293,7 @@ "fieldname": "search_index_fields", "fieldtype": "Small Text", "label": "Search Index Fields", + "mandatory_depends_on": "is_redisearch_enabled", "read_only_depends_on": "eval:!doc.is_redisearch_loaded" }, { @@ -301,13 +302,6 @@ "fieldtype": "Section Break", "label": "Item Search Settings" }, - { - "default": "1", - "fieldname": "show_categories_in_search_autocomplete", - "fieldtype": "Check", - "label": "Show Categories in Search Autocomplete", - "read_only_depends_on": "eval:!doc.is_redisearch_loaded" - }, { "default": "0", "fieldname": "is_redisearch_loaded", @@ -365,12 +359,19 @@ "fieldname": "show_price_in_quotation", "fieldtype": "Check", "label": "Show Price in Quotation" + }, + { + "default": "0", + "fieldname": "is_redisearch_enabled", + "fieldtype": "Check", + "label": "Enable Redisearch", + "read_only_depends_on": "eval:!doc.is_redisearch_loaded" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-09-02 14:02:44.785824", + "modified": "2022-04-01 18:35:56.106756", "modified_by": "Administrator", "module": "E-commerce", "name": "E Commerce Settings", @@ -389,5 +390,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py index 881d833eb01..f85667e1b20 100644 --- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py +++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py @@ -9,6 +9,7 @@ from frappe.utils import comma_and, flt, unique from erpnext.e_commerce.redisearch_utils import ( create_website_items_index, + define_autocomplete_dictionary, get_indexable_web_fields, is_search_module_loaded, ) @@ -21,6 +22,8 @@ class ShoppingCartSetupError(frappe.ValidationError): class ECommerceSettings(Document): def onload(self): self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series") + + # flag >> if redisearch is installed and loaded self.is_redisearch_loaded = is_search_module_loaded() def validate(self): @@ -34,6 +37,20 @@ class ECommerceSettings(Document): frappe.clear_document_cache("E Commerce Settings", "E Commerce Settings") + self.is_redisearch_enabled_pre_save = frappe.db.get_single_value( + "E Commerce Settings", "is_redisearch_enabled" + ) + + def after_save(self): + self.create_redisearch_indexes() + + def create_redisearch_indexes(self): + # if redisearch is enabled (value changed) create indexes and dictionary + value_changed = self.is_redisearch_enabled != self.is_redisearch_enabled_pre_save + if self.is_redisearch_loaded and self.is_redisearch_enabled and value_changed: + define_autocomplete_dictionary() + create_website_items_index() + def validate_field_filters(self): if not (self.enable_field_filters and self.filter_fields): return diff --git a/erpnext/e_commerce/redisearch_utils.py b/erpnext/e_commerce/redisearch_utils.py index 82829bf1eff..f2dd796f2c5 100644 --- a/erpnext/e_commerce/redisearch_utils.py +++ b/erpnext/e_commerce/redisearch_utils.py @@ -1,8 +1,12 @@ -# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt +import json + import frappe +from frappe import _ from frappe.utils.redis_wrapper import RedisWrapper +from redis import ResponseError from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField WEBSITE_ITEM_INDEX = "website_items_index" @@ -22,6 +26,12 @@ def get_indexable_web_fields(): return [df.fieldname for df in valid_fields] +def is_redisearch_enabled(): + "Return True only if redisearch is loaded and enabled." + is_redisearch_enabled = frappe.db.get_single_value("E Commerce Settings", "is_redisearch_enabled") + return is_search_module_loaded() and is_redisearch_enabled + + def is_search_module_loaded(): try: cache = frappe.cache() @@ -32,14 +42,14 @@ def is_search_module_loaded(): ) return "search" in parsed_output except Exception: - return False + return False # handling older redis versions -def if_redisearch_loaded(function): - "Decorator to check if Redisearch is loaded." +def if_redisearch_enabled(function): + "Decorator to check if Redisearch is enabled." def wrapper(*args, **kwargs): - if is_search_module_loaded(): + if is_redisearch_enabled(): func = function(*args, **kwargs) return func return @@ -51,22 +61,25 @@ def make_key(key): return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8") -@if_redisearch_loaded +@if_redisearch_enabled def create_website_items_index(): "Creates Index Definition." # CREATE index client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache()) - # DROP if already exists try: - client.drop_index() - except Exception: + client.drop_index() # drop if already exists + except ResponseError: + # will most likely raise a ResponseError if index does not exist + # ignore and create index pass + except Exception: + raise_redisearch_error() idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)]) - # Based on e-commerce settings + # Index fields mentioned in e-commerce settings idx_fields = frappe.db.get_single_value("E Commerce Settings", "search_index_fields") idx_fields = idx_fields.split(",") if idx_fields else [] @@ -91,20 +104,20 @@ def to_search_field(field): return TextField(field) -@if_redisearch_loaded +@if_redisearch_enabled def insert_item_to_index(website_item_doc): # Insert item to index key = get_cache_key(website_item_doc.name) cache = frappe.cache() web_item = create_web_item_map(website_item_doc) - for k, v in web_item.items(): - super(RedisWrapper, cache).hset(make_key(key), k, v) + for field, value in web_item.items(): + super(RedisWrapper, cache).hset(make_key(key), field, value) insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name) -@if_redisearch_loaded +@if_redisearch_enabled def insert_to_name_ac(web_name, doc_name): ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache()) ac.add_suggestions(Suggestion(web_name, payload=doc_name)) @@ -114,20 +127,20 @@ def create_web_item_map(website_item_doc): fields_to_index = get_fields_indexed() web_item = {} - for f in fields_to_index: - web_item[f] = website_item_doc.get(f) or "" + for field in fields_to_index: + web_item[field] = website_item_doc.get(field) or "" return web_item -@if_redisearch_loaded +@if_redisearch_enabled def update_index_for_item(website_item_doc): # Reinsert to Cache insert_item_to_index(website_item_doc) define_autocomplete_dictionary() -@if_redisearch_loaded +@if_redisearch_enabled def delete_item_from_index(website_item_doc): cache = frappe.cache() key = get_cache_key(website_item_doc.name) @@ -135,13 +148,13 @@ def delete_item_from_index(website_item_doc): try: cache.delete(key) except Exception: - return False + raise_redisearch_error() delete_from_ac_dict(website_item_doc) return True -@if_redisearch_loaded +@if_redisearch_enabled def delete_from_ac_dict(website_item_doc): """Removes this items's name from autocomplete dictionary""" cache = frappe.cache() @@ -149,40 +162,60 @@ def delete_from_ac_dict(website_item_doc): name_ac.delete(website_item_doc.web_item_name) -@if_redisearch_loaded +@if_redisearch_enabled def define_autocomplete_dictionary(): - """Creates an autocomplete search dictionary for `name`. - Also creats autocomplete dictionary for `categories` if - checked in E Commerce Settings""" + """ + Defines/Redefines an autocomplete search dictionary for Website Item Name. + Also creats autocomplete dictionary for Published Item Groups. + """ cache = frappe.cache() - name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) - cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) - - ac_categories = frappe.db.get_single_value( - "E Commerce Settings", "show_categories_in_search_autocomplete" - ) + item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) + item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) # Delete both autocomplete dicts try: cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE)) cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE)) except Exception: - return False + raise_redisearch_error() + create_items_autocomplete_dict(autocompleter=item_ac) + create_item_groups_autocomplete_dict(autocompleter=item_group_ac) + + +@if_redisearch_enabled +def create_items_autocomplete_dict(autocompleter): + "Add items as suggestions in Autocompleter." items = frappe.get_all( "Website Item", fields=["web_item_name", "item_group"], filters={"published": 1} ) for item in items: - name_ac.add_suggestions(Suggestion(item.web_item_name)) - if ac_categories and item.item_group: - cat_ac.add_suggestions(Suggestion(item.item_group)) - - return True + autocompleter.add_suggestions(Suggestion(item.web_item_name)) -@if_redisearch_loaded +@if_redisearch_enabled +def create_item_groups_autocomplete_dict(autocompleter): + "Add item groups with weightage as suggestions in Autocompleter." + published_item_groups = frappe.get_all( + "Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1} + ) + if not published_item_groups: + return + + for item_group in published_item_groups: + payload = json.dumps({"name": item_group.name, "route": item_group.route}) + autocompleter.add_suggestions( + Suggestion( + string=item_group.name, + score=frappe.utils.flt(item_group.weightage) or 1.0, + payload=payload, # additional info that can be retrieved later + ) + ) + + +@if_redisearch_enabled def reindex_all_web_items(): items = frappe.get_all("Website Item", fields=get_fields_indexed(), filters={"published": True}) @@ -191,8 +224,8 @@ def reindex_all_web_items(): web_item = create_web_item_map(item) key = make_key(get_cache_key(item.name)) - for k, v in web_item.items(): - super(RedisWrapper, cache).hset(key, k, v) + for field, value in web_item.items(): + super(RedisWrapper, cache).hset(key, field, value) def get_cache_key(name): @@ -210,7 +243,12 @@ def get_fields_indexed(): return fields_to_index -# TODO: Remove later -# # Figure out a way to run this at startup -define_autocomplete_dictionary() -create_website_items_index() +def raise_redisearch_error(): + "Create an Error Log and raise error." + traceback = frappe.get_traceback() + log = frappe.log_error(traceback, frappe._("Redisearch Error")) + log_link = frappe.utils.get_link_to_form("Error Log", log.name) + + frappe.throw( + msg=_("Something went wrong. Check {0}").format(log_link), title=_("Redisearch Error") + ) diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py index 77a749ef9eb..3ed056f55e7 100644 --- a/erpnext/templates/pages/product_search.py +++ b/erpnext/templates/pages/product_search.py @@ -1,6 +1,8 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt +import json + import frappe from frappe.utils import cint, cstr from redisearch import AutoCompleter, Client, Query @@ -9,7 +11,7 @@ from erpnext.e_commerce.redisearch_utils import ( WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE, - is_search_module_loaded, + is_redisearch_enabled, make_key, ) from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website @@ -74,8 +76,8 @@ def search(query): def product_search(query, limit=10, fuzzy_search=True): search_results = {"from_redisearch": True, "results": []} - if not is_search_module_loaded(): - # Redisearch module not loaded + if not is_redisearch_enabled(): + # Redisearch module not enabled search_results["from_redisearch"] = False search_results["results"] = get_product_data(query, 0, limit) return search_results @@ -86,6 +88,8 @@ def product_search(query, limit=10, fuzzy_search=True): red = frappe.cache() query = clean_up_query(query) + # TODO: Check perf/correctness with Suggestions & Query vs only Query + # TODO: Use Levenshtein Distance in Query (max=3) ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red) client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red) suggestions = ac.get_suggestions( @@ -121,8 +125,8 @@ def convert_to_dict(redis_search_doc): def get_category_suggestions(query): search_results = {"results": []} - if not is_search_module_loaded(): - # Redisearch module not loaded, query db + if not is_redisearch_enabled(): + # Redisearch module not enabled, query db categories = frappe.db.get_all( "Item Group", filters={"name": ["like", "%{0}%".format(query)], "show_in_website": 1}, @@ -135,8 +139,10 @@ def get_category_suggestions(query): return search_results ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) - suggestions = ac.get_suggestions(query, num=10) + suggestions = ac.get_suggestions(query, num=10, with_payloads=True) - search_results["results"] = [s.string for s in suggestions] + results = [json.loads(s.payload) for s in suggestions] + + search_results["results"] = results return search_results