From 43ad2fed631a689cce9b88369f959f6e7049e8d7 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 13:05:30 +0530 Subject: [PATCH 1/8] fix: custom delimiters (cherry picked from commit 13fb560401a89e64b0ff6400b0477314499bdfa8) # Conflicts: # erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json --- .../bank_statement_import.json | 20 +++++++++++++++++++ .../bank_statement_import.py | 2 ++ 2 files changed, 22 insertions(+) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json index eede3bdc6d1..fac19fd2d28 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json @@ -16,6 +16,8 @@ "html_5", "import_file", "download_template", + "custom_delimiters", + "delimiter_options", "status", "template_options", "import_warnings_section", @@ -198,11 +200,29 @@ "fieldtype": "Code", "label": "Statement Import Log", "options": "JSON" + }, + { + "default": "0", + "fieldname": "custom_delimiters", + "fieldtype": "Check", + "label": "Custom delimiters" + }, + { + "default": ",;\\t|", + "depends_on": "custom_delimiters", + "description": "If your CSV uses a different delimiter, add that character here, ensuring no spaces or additional characters are included.", + "fieldname": "delimiter_options", + "fieldtype": "Data", + "label": "Delimiter options" } ], "hide_toolbar": 1, "links": [], +<<<<<<< HEAD "modified": "2022-09-07 11:11:40.293317", +======= + "modified": "2024-06-25 15:34:35.214946", +>>>>>>> 13fb560401 (fix: custom delimiters) "modified_by": "Administrator", "module": "Accounts", "name": "Bank Statement Import", diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py index 0d19e212cba..9b0989f5b23 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py @@ -31,6 +31,8 @@ class BankStatementImport(DataImport): bank: DF.Link | None bank_account: DF.Link company: DF.Link + custom_delimiters: DF.Check + delimiter_options: DF.Data | None google_sheets_url: DF.Data | None import_file: DF.Attach | None import_type: DF.Literal["", "Insert New Records", "Update Existing Records"] From 2f0b97d91b6027126978642090f0e30686181ee6 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 14:27:04 +0530 Subject: [PATCH 2/8] fix: remove deprecated field "statement_import_log" (cherry picked from commit b20240942831d1118d00ca8762ebd4e0ff18f684) # Conflicts: # erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json --- .../bank_statement_import/bank_statement_import.json | 11 ++++------- .../bank_statement_import/bank_statement_import.py | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json index fac19fd2d28..4d9721f771d 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json @@ -26,7 +26,6 @@ "section_import_preview", "import_preview", "import_log_section", - "statement_import_log", "show_failed_logs", "import_log_preview", "reference_doctype", @@ -195,12 +194,6 @@ "fieldname": "column_break_4", "fieldtype": "Column Break" }, - { - "fieldname": "statement_import_log", - "fieldtype": "Code", - "label": "Statement Import Log", - "options": "JSON" - }, { "default": "0", "fieldname": "custom_delimiters", @@ -218,11 +211,15 @@ ], "hide_toolbar": 1, "links": [], +<<<<<<< HEAD <<<<<<< HEAD "modified": "2022-09-07 11:11:40.293317", ======= "modified": "2024-06-25 15:34:35.214946", >>>>>>> 13fb560401 (fix: custom delimiters) +======= + "modified": "2024-06-25 16:33:50.675848", +>>>>>>> b202409428 (fix: remove deprecated field "statement_import_log") "modified_by": "Administrator", "module": "Accounts", "name": "Bank Statement Import", diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py index 9b0989f5b23..0311f8b1bcb 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py @@ -39,7 +39,6 @@ class BankStatementImport(DataImport): mute_emails: DF.Check reference_doctype: DF.Link show_failed_logs: DF.Check - statement_import_log: DF.Code | None status: DF.Literal["Pending", "Success", "Partial Success", "Error"] submit_after_import: DF.Check template_options: DF.Code | None From 71311ffd624560994f7f558c3af73a66d4db38c6 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 14:27:28 +0530 Subject: [PATCH 3/8] fix: import status (cherry picked from commit 07f68884e0c09fdcf769dc7f1d0cc1a03921ef01) --- .../bank_statement_import.js | 102 ++++++++++-------- .../bank_statement_import.py | 28 +++++ 2 files changed, 85 insertions(+), 45 deletions(-) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js index 3dfbf701060..29cf5189d82 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js @@ -120,52 +120,64 @@ frappe.ui.form.on("Bank Statement Import", { }, show_import_status(frm) { - let import_log = JSON.parse(frm.doc.statement_import_log || "[]"); - let successful_records = import_log.filter((log) => log.success); - let failed_records = import_log.filter((log) => !log.success); - if (successful_records.length === 0) return; + frappe.call({ + method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_status", + args: { + docname: frm.doc.name, + }, + callback: function (r) { + let successful_records = cint(r.message.success); + let failed_records = cint(r.message.failed); + let total_records = cint(r.message.total_records); - let message; - if (failed_records.length === 0) { - let message_args = [successful_records.length]; - if (frm.doc.import_type === "Insert New Records") { - message = - successful_records.length > 1 - ? __("Successfully imported {0} records.", message_args) - : __("Successfully imported {0} record.", message_args); - } else { - message = - successful_records.length > 1 - ? __("Successfully updated {0} records.", message_args) - : __("Successfully updated {0} record.", message_args); - } - } else { - let message_args = [successful_records.length, import_log.length]; - if (frm.doc.import_type === "Insert New Records") { - message = - successful_records.length > 1 - ? __( - "Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ) - : __( - "Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ); - } else { - message = - successful_records.length > 1 - ? __( - "Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ) - : __( - "Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", - message_args - ); - } - } - frm.dashboard.set_headline(message); + if (!total_records) { + return; + } + + let message; + if (failed_records === 0) { + let message_args = [successful_records]; + if (frm.doc.import_type === "Insert New Records") { + message = + successful_records > 1 + ? __("Successfully imported {0} records.", message_args) + : __("Successfully imported {0} record.", message_args); + } else { + message = + successful_records > 1 + ? __("Successfully updated {0} records.", message_args) + : __("Successfully updated {0} record.", message_args); + } + } else { + let message_args = [successful_records, total_records]; + if (frm.doc.import_type === "Insert New Records") { + message = + successful_records > 1 + ? __( + "Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); + } else { + message = + successful_records > 1 + ? __( + "Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); + } + } + + frm.dashboard.set_headline(message); + }, + }); }, show_report_error_button(frm) { diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py index 0311f8b1bcb..06f742afc19 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py @@ -242,6 +242,34 @@ def write_xlsx(data, sheet_name, wb=None, column_widths=None, file_path=None): return True +@frappe.whitelist() +def get_import_status(docname): + import_status = {} + + data_import = frappe.get_doc("Bank Statement Import", docname) + import_status["status"] = data_import.status + + logs = frappe.get_all( + "Data Import Log", + fields=["count(*) as count", "success"], + filters={"data_import": docname}, + group_by="success", + ) + + total_payload_count = 0 + + for log in logs: + total_payload_count += log.get("count", 0) + if log.get("success"): + import_status["success"] = log.get("count") + else: + import_status["failed"] = log.get("count") + + import_status["total_records"] = total_payload_count + + return import_status + + @frappe.whitelist() def upload_bank_statement(**args): args = frappe._dict(args) From 62aac8bb85b3fc28d5db9074ccfc2a03a3d76fb9 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 14:47:23 +0530 Subject: [PATCH 4/8] fix: import log preview (cherry picked from commit 1ad264de48216889d1c1a5a83d0cd773070049d5) --- .../bank_statement_import.js | 142 +++++++++++------- .../bank_statement_import.py | 13 ++ 2 files changed, 99 insertions(+), 56 deletions(-) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js index 29cf5189d82..d27a3fe1d33 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js @@ -299,7 +299,7 @@ frappe.ui.form.on("Bank Statement Import", { // method: 'frappe.core.doctype.data_import.data_import.get_preview_from_template', show_import_preview(frm, preview_data) { - let import_log = JSON.parse(frm.doc.statement_import_log || "[]"); + let import_log = preview_data.import_log; if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) { frm.import_preview.preview_data = preview_data; @@ -413,49 +413,50 @@ frappe.ui.form.on("Bank Statement Import", { frm.trigger("show_import_log"); }, - show_import_log(frm) { - let import_log = JSON.parse(frm.doc.statement_import_log || "[]"); - let logs = import_log; - frm.toggle_display("import_log", false); - frm.toggle_display("import_log_section", logs.length > 0); + render_import_log(frm) { + frappe.call({ + method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_logs", + args: { + docname: frm.doc.name, + }, + callback: function (r) { + let logs = r.message; - if (logs.length === 0) { - frm.get_field("import_log_preview").$wrapper.empty(); - return; - } + if (logs.length === 0) return; - let rows = logs - .map((log) => { - let html = ""; - if (log.success) { - if (frm.doc.import_type === "Insert New Records") { - html = __("Successfully imported {0}", [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}`, - ]); - } else { - html = __("Successfully updated {0}", [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}`, - ]); - } - } else { - let messages = log.messages - .map(JSON.parse) - .map((m) => { - let title = m.title ? `${m.title}` : ""; - let message = m.message ? `
${m.message}
` : ""; - return title + message; - }) - .join(""); - let id = frappe.dom.get_unique_id(); - html = `${messages} + frm.toggle_display("import_log_section", true); + + let rows = logs + .map((log) => { + let html = ""; + if (log.success) { + if (frm.doc.import_type === "Insert New Records") { + html = __("Successfully imported {0}", [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}`, + ]); + } else { + html = __("Successfully updated {0}", [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}`, + ]); + } + } else { + let messages = JSON.parse(log.messages || "[]") + .map((m) => { + let title = m.title ? `${m.title}` : ""; + let message = m.message ? `
${m.message}
` : ""; + return title + message; + }) + .join(""); + let id = frappe.dom.get_unique_id(); + html = `${messages} @@ -464,16 +465,16 @@ frappe.ui.form.on("Bank Statement Import", {
${log.exception}
`; - } - let indicator_color = log.success ? "green" : "red"; - let title = log.success ? __("Success") : __("Failure"); + } + let indicator_color = log.success ? "green" : "red"; + let title = log.success ? __("Success") : __("Failure"); - if (frm.doc.show_failed_logs && log.success) { - return ""; - } + if (frm.doc.show_failed_logs && log.success) { + return ""; + } - return ` - ${log.row_indexes.join(", ")} + return ` + ${JSON.parse(log.row_indexes).join(", ")}
${title}
@@ -481,16 +482,16 @@ frappe.ui.form.on("Bank Statement Import", { ${html} `; - }) - .join(""); + }) + .join(""); - if (!rows && frm.doc.show_failed_logs) { - rows = ` + if (!rows && frm.doc.show_failed_logs) { + rows = ` ${__("No failed logs")} `; - } + } - frm.get_field("import_log_preview").$wrapper.html(` + frm.get_field("import_log_preview").$wrapper.html(` @@ -500,5 +501,34 @@ frappe.ui.form.on("Bank Statement Import", { ${rows}
${__("Row Number")}
`); + }, + }); + }, + + show_import_log(frm) { + frm.toggle_display("import_log_section", false); + + if (frm.is_new() || frm.import_in_progress) { + return; + } + + frappe.call({ + method: "frappe.client.get_count", + args: { + doctype: "Data Import Log", + filters: { + data_import: frm.doc.name, + }, + }, + callback: function (r) { + let count = r.message; + if (count < 5000) { + frm.trigger("render_import_log"); + } else { + frm.toggle_display("import_log_section", false); + frm.add_custom_button(__("Export Import Log"), () => frm.trigger("export_import_log")); + } + }, + }); }, }); diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py index 06f742afc19..03979c42592 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py @@ -270,6 +270,19 @@ def get_import_status(docname): return import_status +@frappe.whitelist() +def get_import_logs(docname: str): + frappe.has_permission("Bank Statement Import") + + return frappe.get_all( + "Data Import Log", + fields=["success", "docname", "messages", "exception", "row_indexes"], + filters={"data_import": docname}, + limit_page_length=5000, + order_by="log_index", + ) + + @frappe.whitelist() def upload_bank_statement(**args): args = frappe._dict(args) From 4eb251b59ac5e26b06b0640a77bd6fe52f3e491b Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 14:55:51 +0530 Subject: [PATCH 5/8] fix: download_import_log if rows are greater than 5000 (cherry picked from commit 2112d8f772e1536f15073af123211b7b78d02460) --- .../bank_statement_import/bank_statement_import.js | 9 +++++++++ .../bank_statement_import/bank_statement_import.py | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js index d27a3fe1d33..d64d71e7312 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js @@ -338,6 +338,15 @@ frappe.ui.form.on("Bank Statement Import", { ); }, + export_import_log(frm) { + open_url_post( + "/api/method/erpnext.accounts.doctype.bank_statement_import.bank_statement_import.download_import_log", + { + data_import_name: frm.doc.name, + } + ); + }, + show_import_warnings(frm, preview_data) { let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || "[]"); diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py index 03979c42592..fd18819187e 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.py @@ -121,6 +121,11 @@ def download_errored_template(data_import_name): data_import.export_errored_rows() +@frappe.whitelist() +def download_import_log(data_import_name): + return frappe.get_doc("Bank Statement Import", data_import_name).download_import_log() + + def parse_data_from_template(raw_data): data = [] From a04938d5ae9f9312757568ee4e01e18c9f4df901 Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 15:04:21 +0530 Subject: [PATCH 6/8] fix: field position (cherry picked from commit 98ad01c7360803435ee0f6f809b4681e2fe01d48) # Conflicts: # erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json --- .../bank_statement_import/bank_statement_import.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json index 4d9721f771d..a2eec24023b 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json @@ -11,13 +11,13 @@ "bank_account", "bank", "column_break_4", + "custom_delimiters", + "delimiter_options", "google_sheets_url", "refresh_google_sheet", "html_5", "import_file", "download_template", - "custom_delimiters", - "delimiter_options", "status", "template_options", "import_warnings_section", @@ -212,6 +212,7 @@ "hide_toolbar": 1, "links": [], <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD "modified": "2022-09-07 11:11:40.293317", ======= @@ -220,6 +221,9 @@ ======= "modified": "2024-06-25 16:33:50.675848", >>>>>>> b202409428 (fix: remove deprecated field "statement_import_log") +======= + "modified": "2024-06-25 17:32:07.658250", +>>>>>>> 98ad01c736 (fix: field position) "modified_by": "Administrator", "module": "Accounts", "name": "Bank Statement Import", From 57896a8f9936f4716d9282df04e890ab5201e44e Mon Sep 17 00:00:00 2001 From: ljain112 Date: Tue, 25 Jun 2024 17:48:10 +0530 Subject: [PATCH 7/8] fix: changes as per review (cherry picked from commit ba08b2e8e808c154b155995bd93498004c2ff926) --- .../doctype/bank_statement_import/bank_statement_import.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js index d64d71e7312..7361cb2fc1c 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.js @@ -120,6 +120,8 @@ frappe.ui.form.on("Bank Statement Import", { }, show_import_status(frm) { + if (frm.doc.status == "Pending") return; + frappe.call({ method: "erpnext.accounts.doctype.bank_statement_import.bank_statement_import.get_import_status", args: { From f281e064f2004745f730cf919f719e0489186be9 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Tue, 9 Jul 2024 10:39:02 +0530 Subject: [PATCH 8/8] chore: resolve conflict --- .../bank_statement_import/bank_statement_import.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json index a2eec24023b..3210457c645 100644 --- a/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json +++ b/erpnext/accounts/doctype/bank_statement_import/bank_statement_import.json @@ -211,19 +211,7 @@ ], "hide_toolbar": 1, "links": [], -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD - "modified": "2022-09-07 11:11:40.293317", -======= - "modified": "2024-06-25 15:34:35.214946", ->>>>>>> 13fb560401 (fix: custom delimiters) -======= - "modified": "2024-06-25 16:33:50.675848", ->>>>>>> b202409428 (fix: remove deprecated field "statement_import_log") -======= "modified": "2024-06-25 17:32:07.658250", ->>>>>>> 98ad01c736 (fix: field position) "modified_by": "Administrator", "module": "Accounts", "name": "Bank Statement Import",