perf: new column Posting Datetime in SLE to optimize stock ledger related queries (backport #39800)

erpnext/accounts/report/gross_profit/gross_profit.py

@@ -965,7 +965,7 @@ def get_stock_ledger_entries(self, item_code, warehouse):
 						& (sle.is_cancelled == 0)
 					)
 					.orderby(sle.item_code)
-					.orderby(sle.warehouse, sle.posting_date, sle.posting_time, sle.creation, order=Order.desc)
+					.orderby(sle.warehouse, sle.posting_datetime, sle.creation, order=Order.desc)
 					.run(as_dict=True)
 				)
 

erpnext/accounts/utils.py

@@ -1372,8 +1372,7 @@ def sort_stock_vouchers_by_posting_date(
 		.select(sle.voucher_type, sle.voucher_no, sle.posting_date, sle.posting_time, sle.creation)
 		.where((sle.is_cancelled == 0) & (sle.voucher_no.isin(voucher_nos)))
 		.groupby(sle.voucher_type, sle.voucher_no)
-		.orderby(sle.posting_date)
-		.orderby(sle.posting_time)
+		.orderby(sle.posting_datetime)
 		.orderby(sle.creation)
 	).run(as_dict=True)
 	sorted_vouchers = [(sle.voucher_type, sle.voucher_no) for sle in sles]

erpnext/manufacturing/doctype/bom/bom.py

@@ -978,8 +978,7 @@ def get_valuation_rate(data):
 			frappe.qb.from_(sle)
 			.select(sle.valuation_rate)
 			.where((sle.item_code == item_code) & (sle.valuation_rate > 0) & (sle.is_cancelled == 0))
-			.orderby(sle.posting_date, order=frappe.qb.desc)
-			.orderby(sle.posting_time, order=frappe.qb.desc)
+			.orderby(sle.posting_datetime, order=frappe.qb.desc)
 			.orderby(sle.creation, order=frappe.qb.desc)
 			.limit(1)
 		).run(as_dict=True)

erpnext/manufacturing/report/work_order_summary/work_order_summary.py

@@ -58,7 +58,7 @@ def get_data(filters):
 		query_filters["creation"] = ("between", [filters.get("from_date"), filters.get("to_date")])
 
 	data = frappe.get_all(
-		"Work Order", fields=fields, filters=query_filters, order_by="planned_start_date asc", debug=1
+		"Work Order", fields=fields, filters=query_filters, order_by="planned_start_date asc"
 	)
 
 	res = []

erpnext/patches.txt

@@ -274,6 +274,7 @@ erpnext.patches.v14_0.clear_reconciliation_values_from_singles
 
 [post_model_sync]
 execute:frappe.delete_doc_if_exists('Workspace', 'ERPNext Integrations Settings')
+erpnext.patches.v14_0.update_posting_datetime_and_dropped_indexes
 erpnext.patches.v14_0.rename_ongoing_status_in_sla_documents
 erpnext.patches.v14_0.delete_shopify_doctypes
 erpnext.patches.v14_0.delete_healthcare_doctypes
@@ -361,4 +362,4 @@ erpnext.stock.doctype.delivery_note.patches.drop_unused_return_against_index # 2
 erpnext.patches.v14_0.set_maintain_stock_for_bom_item
 execute:frappe.db.set_single_value('E Commerce Settings', 'show_actual_qty', 1)
 erpnext.patches.v14_0.delete_orphaned_asset_movement_item_records
-erpnext.patches.v14_0.remove_cancelled_asset_capitalization_from_asset
+erpnext.patches.v14_0.remove_cancelled_asset_capitalization_from_asset

erpnext/patches/v14_0/update_posting_datetime_and_dropped_indexes.py

@@ -0,0 +1,19 @@
+import frappe
+
+
+def execute():
+	frappe.db.sql(
+		"""
+		UPDATE `tabStock Ledger Entry`
+			SET posting_datetime = timestamp(posting_date, posting_time)
+	"""
+	)
+
+	drop_indexes()
+
+
+def drop_indexes():
+	if not frappe.db.has_index("tabStock Ledger Entry", "posting_sort_index"):
+		return
+
+	frappe.db.sql_ddl("ALTER TABLE `tabStock Ledger Entry` DROP INDEX `posting_sort_index`")

erpnext/stock/doctype/purchase_receipt/test_purchase_receipt.py

@@ -3,7 +3,7 @@
 
 import frappe
 from frappe.tests.utils import FrappeTestCase, change_settings
-from frappe.utils import add_days, cint, cstr, flt, today
+from frappe.utils import add_days, cint, cstr, flt, nowtime, today
 from pypika import functions as fn
 
 import erpnext
@@ -2224,6 +2224,95 @@ def test_pr_billed_amount_against_return_entry(self):
 		pr.reload()
 		self.assertEqual(pr.per_billed, 100)
 
+	def test_sle_qty_after_transaction(self):
+		item = make_item(
+			"_Test Item Qty After Transaction",
+			properties={"is_stock_item": 1, "valuation_method": "FIFO"},
+		).name
+
+		posting_date = today()
+		posting_time = nowtime()
+
+		# Step 1: Create Purchase Receipt
+		pr = make_purchase_receipt(
+			item_code=item,
+			qty=1,
+			rate=100,
+			posting_date=posting_date,
+			posting_time=posting_time,
+			do_not_save=1,
+		)
+
+		for i in range(9):
+			pr.append(
+				"items",
+				{
+					"item_code": item,
+					"qty": 1,
+					"rate": 100,
+					"warehouse": pr.items[0].warehouse,
+					"cost_center": pr.items[0].cost_center,
+					"expense_account": pr.items[0].expense_account,
+					"uom": pr.items[0].uom,
+					"stock_uom": pr.items[0].stock_uom,
+					"conversion_factor": pr.items[0].conversion_factor,
+				},
+			)
+
+		self.assertEqual(len(pr.items), 10)
+		pr.save()
+		pr.submit()
+
+		data = frappe.get_all(
+			"Stock Ledger Entry",
+			fields=["qty_after_transaction", "creation", "posting_datetime"],
+			filters={"voucher_no": pr.name, "is_cancelled": 0},
+			order_by="creation",
+		)
+
+		for index, d in enumerate(data):
+			self.assertEqual(d.qty_after_transaction, 1 + index)
+
+		# Step 2: Create Purchase Receipt
+		pr = make_purchase_receipt(
+			item_code=item,
+			qty=1,
+			rate=100,
+			posting_date=posting_date,
+			posting_time=posting_time,
+			do_not_save=1,
+		)
+
+		for i in range(9):
+			pr.append(
+				"items",
+				{
+					"item_code": item,
+					"qty": 1,
+					"rate": 100,
+					"warehouse": pr.items[0].warehouse,
+					"cost_center": pr.items[0].cost_center,
+					"expense_account": pr.items[0].expense_account,
+					"uom": pr.items[0].uom,
+					"stock_uom": pr.items[0].stock_uom,
+					"conversion_factor": pr.items[0].conversion_factor,
+				},
+			)
+
+		self.assertEqual(len(pr.items), 10)
+		pr.save()
+		pr.submit()
+
+		data = frappe.get_all(
+			"Stock Ledger Entry",
+			fields=["qty_after_transaction", "creation", "posting_datetime"],
+			filters={"voucher_no": pr.name, "is_cancelled": 0},
+			order_by="creation",
+		)
+
+		for index, d in enumerate(data):
+			self.assertEqual(d.qty_after_transaction, 11 + index)
+
 
 def prepare_data_for_internal_transfer():
 	from erpnext.accounts.doctype.sales_invoice.test_sales_invoice import create_internal_supplier

erpnext/stock/doctype/stock_entry/test_stock_entry.py

@@ -1671,24 +1671,22 @@ def test_negative_stock_reco(self):
 		item_code = "Test Negative Item - 001"
 		item_doc = create_item(item_code=item_code, is_stock_item=1, valuation_rate=10)
 
-		make_stock_entry(
+		se1 = make_stock_entry(
 			item_code=item_code,
 			posting_date=add_days(today(), -3),
 			posting_time="00:00:00",
-			purpose="Material Receipt",
+			target="_Test Warehouse - _TC",
 			qty=10,
 			to_warehouse="_Test Warehouse - _TC",
-			do_not_save=True,
 		)
 
-		make_stock_entry(
+		se2 = make_stock_entry(
 			item_code=item_code,
 			posting_date=today(),
 			posting_time="00:00:00",
-			purpose="Material Receipt",
+			source="_Test Warehouse - _TC",
 			qty=8,
 			from_warehouse="_Test Warehouse - _TC",
-			do_not_save=True,
 		)
 
 		sr_doc = create_stock_reconciliation(

erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.json

@@ -11,6 +11,7 @@
   "warehouse",
   "posting_date",
   "posting_time",
+  "posting_datetime",
   "is_adjustment_entry",
   "column_break_6",
   "voucher_type",
@@ -96,7 +97,6 @@
    "oldfieldtype": "Date",
    "print_width": "100px",
    "read_only": 1,
-   "search_index": 1,
    "width": "100px"
   },
   {
@@ -249,7 +249,6 @@
    "options": "Company",
    "print_width": "150px",
    "read_only": 1,
-   "search_index": 1,
    "width": "150px"
   },
   {
@@ -316,6 +315,11 @@
    "fieldname": "is_adjustment_entry",
    "fieldtype": "Check",
    "label": "Is Adjustment Entry"
+  },
+  {
+   "fieldname": "posting_datetime",
+   "fieldtype": "Datetime",
+   "label": "Posting Datetime"
   }
  ],
  "hide_toolbar": 1,
@@ -324,7 +328,7 @@
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2024-03-13 09:56:13.021696",
+ "modified": "2024-02-07 09:18:13.999231",
  "modified_by": "Administrator",
  "module": "Stock",
  "name": "Stock Ledger Entry",

erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py

@@ -52,6 +52,12 @@ def validate(self):
 		self.validate_with_last_transaction_posting_time()
 		self.validate_inventory_dimension_negative_stock()
 
+	def set_posting_datetime(self):
+		from erpnext.stock.utils import get_combine_datetime
+
+		self.posting_datetime = get_combine_datetime(self.posting_date, self.posting_time)
+		self.db_set("posting_datetime", self.posting_datetime)
+
 	def validate_inventory_dimension_negative_stock(self):
 		if self.is_cancelled:
 			return
@@ -122,6 +128,7 @@ def _get_inventory_dimensions(self):
 		return inv_dimension_dict
 
 	def on_submit(self):
+		self.set_posting_datetime()
 		self.check_stock_frozen_date()
 		self.calculate_batch_qty()
 
@@ -293,9 +300,7 @@ def on_cancel(self):
 
 
 def on_doctype_update():
-	frappe.db.add_index(
-		"Stock Ledger Entry", fields=["posting_date", "posting_time"], index_name="posting_sort_index"
-	)
 	frappe.db.add_index("Stock Ledger Entry", ["voucher_no", "voucher_type"])
 	frappe.db.add_index("Stock Ledger Entry", ["batch_no", "item_code", "warehouse"])
 	frappe.db.add_index("Stock Ledger Entry", ["warehouse", "item_code"], "item_warehouse")
+	frappe.db.add_index("Stock Ledger Entry", ["posting_datetime", "creation"])

erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py

@@ -2,6 +2,7 @@
 # See license.txt
 
 import json
+import time
 from uuid import uuid4
 
 import frappe
@@ -1066,7 +1067,7 @@ def ordered_qty_after_transaction():
 				frappe.qb.from_(sle)
 				.select("qty_after_transaction")
 				.where((sle.item_code == item) & (sle.warehouse == warehouse) & (sle.is_cancelled == 0))
-				.orderby(CombineDatetime(sle.posting_date, sle.posting_time))
+				.orderby(sle.posting_datetime)
 				.orderby(sle.creation)
 			).run(pluck=True)
 
@@ -1143,6 +1144,89 @@ def test_timestamp_clash(self):
 		except Exception as e:
 			self.fail("Double processing of qty for clashing timestamp.")
 
+	def test_previous_sle_with_clashed_timestamp(self):
+
+		item = make_item().name
+		warehouse = "_Test Warehouse - _TC"
+
+		reciept1 = make_stock_entry(
+			item_code=item,
+			to_warehouse=warehouse,
+			qty=100,
+			rate=10,
+			posting_date="2021-01-01",
+			posting_time="02:00:00",
+		)
+
+		time.sleep(3)
+
+		reciept2 = make_stock_entry(
+			item_code=item,
+			to_warehouse=warehouse,
+			qty=5,
+			posting_date="2021-01-01",
+			rate=10,
+			posting_time="02:00:00.1234",
+		)
+
+		sle = frappe.get_all(
+			"Stock Ledger Entry",
+			filters={"voucher_no": reciept1.name},
+			fields=["qty_after_transaction", "actual_qty"],
+		)
+		self.assertEqual(sle[0].qty_after_transaction, 100)
+		self.assertEqual(sle[0].actual_qty, 100)
+
+		sle = frappe.get_all(
+			"Stock Ledger Entry",
+			filters={"voucher_no": reciept2.name},
+			fields=["qty_after_transaction", "actual_qty"],
+		)
+		self.assertEqual(sle[0].qty_after_transaction, 105)
+		self.assertEqual(sle[0].actual_qty, 5)
+
+	def test_backdated_sle_with_same_timestamp(self):
+
+		item = make_item().name
+		warehouse = "_Test Warehouse - _TC"
+
+		reciept1 = make_stock_entry(
+			item_code=item,
+			to_warehouse=warehouse,
+			qty=5,
+			posting_date="2021-01-01",
+			rate=10,
+			posting_time="02:00:00.1234",
+		)
+
+		time.sleep(3)
+
+		# backdated entry with same timestamp but different ms part
+		reciept2 = make_stock_entry(
+			item_code=item,
+			to_warehouse=warehouse,
+			qty=100,
+			rate=10,
+			posting_date="2021-01-01",
+			posting_time="02:00:00",
+		)
+
+		sle = frappe.get_all(
+			"Stock Ledger Entry",
+			filters={"voucher_no": reciept1.name},
+			fields=["qty_after_transaction", "actual_qty"],
+		)
+		self.assertEqual(sle[0].qty_after_transaction, 5)
+		self.assertEqual(sle[0].actual_qty, 5)
+
+		sle = frappe.get_all(
+			"Stock Ledger Entry",
+			filters={"voucher_no": reciept2.name},
+			fields=["qty_after_transaction", "actual_qty"],
+		)
+		self.assertEqual(sle[0].qty_after_transaction, 105)
+		self.assertEqual(sle[0].actual_qty, 100)
+
 	@change_settings("System Settings", {"float_precision": 3, "currency_precision": 2})
 	def test_transfer_invariants(self):
 		"""Extact stock value should be transferred."""