✨ Text based Doctype import

.travis.yml

@@ -43,7 +43,7 @@ script:
   - docker-compose -f docker-compose.${DATABASE}.yml ps "erpnext_app" | grep "Up"
   - docker-compose -f docker-compose.${DATABASE}.yml logs "erpnext_web"
   - docker-compose -f docker-compose.${DATABASE}.yml ps "erpnext_web" | grep "Up"
-  - echo 'Wait until test finished (3 minutes)' && sleep 180
+  - echo 'Wait until test finished (5 minutes)' && sleep 300
   - docker-compose -f docker-compose.${DATABASE}.yml logs "sut"
   - docker-compose -f docker-compose.${DATABASE}.yml ps "sut" | grep "Exit 0"
   # Test container restart

CONTRIBUTING.md

@@ -7,7 +7,7 @@ You can report bugs or request new features by creating an [issue](https://githu
 
 Here are a few rules to follow in order to ease code reviews and discussions before maintainers accept and merge your work.
 
-You MUST follow the [The Best of the Best Practices (BOBP) Guide for Python](https://gist.github.com/sloria/7001839). If you don't know about any of them, you should really read the recommendations.
+You MUST follow the [Best of the Best Practices (BOBP) Guide for Python](https://gist.github.com/sloria/7001839). If you don't know about any of them, you should really read the recommendations.
 
 You SHOULD write tests.
 

Dockerfile.test

@@ -5,7 +5,7 @@ ADD .travis/docker_test.sh /docker_test.sh
 RUN set -ex; \
     sudo chmod 755 /docker_test.sh; \
     sudo pip install coverage==4.5.4; \
-    sudo pip install python-coveralls
+    sudo pip install python-coveralls;
 
 EXPOSE 4444
 

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2019 Monogramm
+Copyright (c) 2020 Monogramm
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

erpnext_ocr/erpnext_ocr/doctype/__init__.py

@@ -0,0 +1 @@
+from __future__ import unicode_literals

erpnext_ocr/erpnext_ocr/doctype/ocr_import/__init__.py

@@ -0,0 +1 @@
+from __future__ import unicode_literals

erpnext_ocr/erpnext_ocr/doctype/ocr_import/constants.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+TEST_RESULT_FOR_SI = '''
+ \n\nD. Brawn Manufacture\n\nInvoice no. DVT-AX-345678\n\nPayment date: 03/12/2006\n\n \n\n 
+ \n\n \n\n \n\nReference Designation ty Unit price “Total CHF Sales\nWork\nSERVICE D COMPLETE OVERHAUL 1 5500.00 500,00 220\nSERVICE D REFRESHING COMPLETE CASE 1 380,00 380.00 220\nAND RHODIUM BATH\nExterior parts:\nJO.297.085.FP FLAT GASKET. 1 3.00 3.00 220\nJO.197.075.FP FLAT GASKET 1 4.00 4.00 220\nJO.199.059.08 FLAT ROUND GASKET 1 6.00 600 220\nVi.261.036.8C  W.G.FIXATION SCREWS 10 4.00 40.00 220\nAL465.085.BC WHITE GOLD "FOIL" 1 70.00 70.00 220\nPAIR OF HAND\nLENGTH: 10/13.50MM\nCALIBRE 2868\nSPECIAL DISCOUNT -3003.00 — -3003.00\nDiscount “500.00 “800.00\nTotal CHF 2100.00\nRETURN AFTER REPAIR\n\nNO COMMERCIAL VALUE\n'''
+
+TEST_RESULT_FOR_ITEM = '''
+Series STO-ITEM-.YYYY.-\n\nItem Code fdsa\n\nItem Name fdsa\n\nItem Group Consumable\nDefault Unit of Measure Nos\n\nfdsa\n\nShelf Life In Days\n\nEnd of Life 31-12-2099\nDefault Material Request Purchase\nType\n\nSr UOM\n\n1 Nos\n\nMax Sample Quantity\n\nVariant Based On\n\nSr Company\n\n1 fdsa\n\nIs Purchase Item\nMinimum Order Qty\n\nSafety Stock\nCountry of Origin Russian Federation\n\nIs Sales Item\n\nWeightage\n\nMaintain Stock\n\nInclude Item In Manufacturing\n\nOpening Stock\n\nValuation Rate\nStandard Selling Rate\n\nAllow over delivery or receipt\nupto this percent\n\n0 Weight Per Unit\n\nItem Attribute\n\nDefault Warehouse\n\nStores - F\n\nv Lead Time in days\n\n0 Last Purchase Rate\n\nv Max Discount (%)\nNo of Months\n\nNo of Months\n\nv\n\n0\nRUB 0,00\n\nRUB 123,00\n\n0\n\nConversion Factor\n\n‘l\n
+'''

erpnext_ocr/erpnext_ocr/doctype/ocr_import/ocr_import.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Monogramm and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('OCR Import', {
+	refresh: function(frm) {
+
+	}
+});

erpnext_ocr/erpnext_ocr/doctype/ocr_import/ocr_import.json

@@ -0,0 +1,152 @@
+{
+ "allow_copy": 0,
+ "allow_events_in_timeline": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "autoname": "field:doctype_link",
+ "beta": 0,
+ "creation": "2019-11-25 16:37:26.611197",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+  {
+   "allow_bulk_edit": 0,
+   "allow_in_quick_entry": 0,
+   "allow_on_submit": 0,
+   "bold": 0,
+   "collapsible": 0,
+   "columns": 0,
+   "fetch_if_empty": 0,
+   "fieldname": "doctype_link",
+   "fieldtype": "Link",
+   "hidden": 0,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_filter": 0,
+   "in_global_search": 0,
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Doctype",
+   "length": 0,
+   "no_copy": 0,
+   "options": "DocType",
+   "permlevel": 0,
+   "precision": "",
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "read_only": 0,
+   "remember_last_selected_value": 0,
+   "report_hide": 0,
+   "reqd": 0,
+   "search_index": 0,
+   "set_only_once": 0,
+   "translatable": 0,
+   "unique": 1
+  },
+  {
+   "allow_bulk_edit": 0,
+   "allow_in_quick_entry": 0,
+   "allow_on_submit": 0,
+   "bold": 0,
+   "collapsible": 0,
+   "columns": 0,
+   "fetch_if_empty": 0,
+   "fieldname": "mappings",
+   "fieldtype": "Table",
+   "hidden": 0,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_filter": 0,
+   "in_global_search": 0,
+   "in_list_view": 0,
+   "in_standard_filter": 0,
+   "label": "Mappings",
+   "length": 0,
+   "no_copy": 0,
+   "options": "OCR Import Mapping",
+   "permlevel": 0,
+   "precision": "",
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "read_only": 0,
+   "remember_last_selected_value": 0,
+   "report_hide": 0,
+   "reqd": 0,
+   "search_index": 0,
+   "set_only_once": 0,
+   "translatable": 0,
+   "unique": 0
+  }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2020-01-24 06:38:56.285558",
+ "modified_by": "Administrator",
+ "module": "ERPNext OCR",
+ "name": "OCR Import",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "amend": 0,
+   "cancel": 0,
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "if_owner": 0,
+   "import": 0,
+   "permlevel": 0,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "set_user_permissions": 0,
+   "share": 1,
+   "submit": 0,
+   "write": 1
+  },
+  {
+   "amend": 0,
+   "cancel": 0,
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "if_owner": 0,
+   "import": 0,
+   "permlevel": 0,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "ERPNext OCR Manager",
+   "set_user_permissions": 0,
+   "share": 1,
+   "submit": 0,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "title_field": "doctype_link",
+ "track_changes": 1,
+ "track_seen": 0,
+ "track_views": 0
+}

erpnext_ocr/erpnext_ocr/doctype/ocr_import/ocr_import.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020, Monogramm and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+
+import re
+
+import frappe
+from erpnext_ocr.erpnext_ocr.doctype.ocr_import_mapping.ocr_import_mapping import find_field, generate_child_doctype
+from frappe.model.document import Document
+
+
+class OCRImport(Document):
+    pass
+
+
+@frappe.whitelist()
+def generate_doctype(doctype_import_link, read_result, ignore_mandatory=False, ignore_validate=False):
+    """
+    Generate doctype from raw text.
+    :param ignore_validate: Ignore validation
+    :param ignore_mandatory: Ignore mandatory fields
+    :param doctype_import_link:
+    :param read_result: text from document
+    :return: generated doctype
+    """
+
+    doctype_import_doc = frappe.get_doc("OCR Import", doctype_import_link)
+    generated_doc = frappe.new_doc(doctype_import_link)
+
+    if ignore_mandatory:
+        generated_doc.company = "_Test Company"
+        generated_doc.price_list = "_Test Price List"
+    generated_doc.flags.ignore_mandatory = ignore_mandatory
+    generated_doc.flags.ignore_validate = ignore_validate
+
+    list_with_errors = []
+    list_with_table_values = []
+
+    for field in doctype_import_doc.mappings:
+        try:
+            found_field = find_field(field, read_result)
+            if found_field is None:
+                frappe.throw(
+                    frappe._("Cannot find field {0} in text").format(field.field))
+            if field.value_type == "Table":
+                iter_of_str = re.finditer(field.regexp, read_result)
+                for item_match in iter_of_str:
+                    raw_table_doc = generated_doc.append(field.field)
+                    raw_table_doc.flags.ignore_mandatory = ignore_mandatory
+                    item_str = item_match.group()
+                    table_doc = generate_child_doctype(field.link_to_child_doc,
+                                                       item_str,
+                                                       raw_table_doc)
+                    list_with_table_values.append(table_doc)
+                    generated_doc.__dict__[field.field] = list_with_table_values
+            elif field.value_type == "Date":
+                generated_doc.__dict__[field.field] = frappe.utils.get_datetime(found_field)
+            else:
+                generated_doc.__dict__[field.field] = found_field
+        except KeyError:
+            list_with_errors.append("Field {} doesn't exist in doctype".
+                                    format(doctype_import_doc))
+    if list_with_errors:
+        frappe.throw(list_with_errors)
+    try:
+        generated_doc.set_new_name()
+        generated_doc.insert()
+    except frappe.exceptions.DuplicateEntryError:
+        frappe.throw("Generated doc already exists")
+    return generated_doc

erpnext_ocr/erpnext_ocr/doctype/ocr_import/test_ocr_import.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: OCR Import", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new OCR Import
+		() => frappe.tests.make('OCR Import', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});