Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Save dom of web and variables to server #482

Merged
merged 13 commits into from
Aug 12, 2024
Merged

Large diffs are not rendered by default.

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion Apps/Web/AI_Recorder_2/dist/background/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@
"stepName": "Step_name",
"url": "",
"apiKey": "",
"jwtKey": ""
"jwtKey": "",
"nodeId": ""
}
2 changes: 1 addition & 1 deletion Apps/Web/AI_Recorder_2/dist/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<!-- <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous"> -->
<!-- <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script> -->

<script type="module" crossorigin src="/assets/index-ZxKYERgc.js"></script>
<script type="module" crossorigin src="/assets/index-iqPt-ocH.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-CrwDEiCX.css">
</head>
<body>
Expand Down
3 changes: 2 additions & 1 deletion Apps/Web/AI_Recorder_2/public/background/data.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@
"stepName": "Step_name",
"url": "",
"apiKey": "",
"jwtKey": ""
"jwtKey": "",
"nodeId": ""
}
4 changes: 3 additions & 1 deletion Apps/Web/AI_Recorder_2/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,9 @@ function App() {
TC_Id: result.meta_data.testNo,
step_sequence: result.meta_data.stepNo,
step_data: JSON.stringify(actions.map(action => {
return action.main;
return action.main.map((row, rowIndex) => {
return [...row, rowIndex + 1];
});
})),
step_id: result.meta_data.stepId,
dataset_name: JSON.stringify(actions.map((action, idx) => {
Expand Down
26 changes: 20 additions & 6 deletions Apps/Web/aiplugin/background.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,18 @@ const activeIcon = 'active-64.png';
const defaultIcon = 'small_logo.png';
var zeuz_url;
var zeuz_key;
var zeuz_node_id;

fetch("data.json")
.then(Response => Response.json())
.then(data => {
zeuz_url = data.zeuz_url;
zeuz_key = data.zeuz_key;
console.log(data);
console.log(zeuz_url);
console.log(zeuz_key);
zeuz_url = data.url;
zeuz_key = data.apiKey;
zeuz_node_id = data.nodeId;
browserAppData.storage.local.set({
url: zeuz_url,
key: zeuz_key,
nodeId: zeuz_node_id,
},
function() {
console.log("Logged in successfully!");
Expand Down Expand Up @@ -137,7 +137,6 @@ browserAppData.runtime.onMessage.addListener(
function(request, sender, sendResponse) {
if (request.apiName == 'ai_record_single_action') {
var url = `${zeuz_url}/ai_record_single_action/`
console.log("zeuz_key", zeuz_key)
fetch(url, {
method: "POST",
headers: {
Expand All @@ -149,6 +148,21 @@ browserAppData.runtime.onMessage.addListener(
.then(response => response.json())
.then(text => {console.log(text);sendResponse(text);})

var url = `${zeuz_url}/node_ai_contents/`
fetch(url, {
method: "POST",
headers: {
// "Content-Type": "application/json",
"X-Api-Key": zeuz_key,
},
body: JSON.stringify({
"dom_web": {"dom": request.html},
"node_id": zeuz_node_id
}),
})
.then(response => response.json())
.then(text => {console.log(text);sendResponse(text);})

return true; // Will respond asynchronously.
}
}
Expand Down
10 changes: 8 additions & 2 deletions Apps/Web/aiplugin/data.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
{
"zeuz_url": "__ZeuZ__UrL_maPP",
"zeuz_key": "__ZeuZ__KeY_maPP"
"testNo": "TEST-0000",
"testName": "Test_name",
"stepNo": 0,
"stepName": "Step_name",
"url": "",
"apiKey": "",
"jwtKey": "",
"nodeId": ""
}
8 changes: 5 additions & 3 deletions Apps/Web/aiplugin/inspect.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,11 @@ class Inspector {

}

async function send_data(server_url, api_key, data, modal_id) {
async function send_data(server_url, api_key, data, modal_id, refinedHtml) {
browserAppData.runtime.sendMessage({
apiName: 'ai_record_single_action',
data: data,
html: refinedHtml,
},
response => {
insert_modal_text(response, modal_id);
Expand Down Expand Up @@ -150,7 +151,7 @@ class Inspector {
"action_type": "selenium"
});

send_data(server_url, api_key, data, this.modalNode);
send_data(server_url, api_key, data, this.modalNode, refinedHtml);

});
// remove zeuz attribute
Expand Down Expand Up @@ -216,12 +217,13 @@ class Inspector {

refinedHtml = refinedHtml.replace(/[^\x00-\x7F]/g, '');
refinedHtml = refinedHtml.replace(/[\x00-\x1F\x7F]/g, '');
refinedHtml = refinedHtml.replace(/\s+/g, ' ').replace(/>\s+</g, '><');
var data = JSON.stringify({
"page_src": refinedHtml,
"action_type": "selenium"
});

send_data(server_url, api_key, data, this.modalNode);
send_data(server_url, api_key, data, this.modalNode, refinedHtml);

});
// remove zeuz attribute
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,6 @@
step_exit_fail_called = False
step_exit_pass_called = False

# Get node ID and set as a Shared Variable
machineInfo = CommonUtil.MachineInfo() # Create instance
node_id = machineInfo.getLocalUser() # Get Username+Node ID
sr.Set_Shared_Variables(
"node_id", node_id, protected=True
) # Save as protected shared variable

from pathlib import Path
if os.path.exists(Path(__file__).parent.parent.parent.parent / "bypass.json"):
bypass_exist = True
Expand Down
18 changes: 8 additions & 10 deletions Framework/Built_In_Automation/Web/Selenium/BuiltInFunctions.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,16 +457,6 @@ def set_extension_variables():
url = ConfigModule.get_config_value("Authentication", "server_address").strip()
apiKey = ConfigModule.get_config_value("Authentication", "api-key").strip()
jwtKey = CommonUtil.jwt_token.strip()
with open(Path(aiplugin_path) / "data.json", "w") as file:
json.dump({
"zeuz_url": url,
"zeuz_key": apiKey
}, file, indent=4)

except:
return CommonUtil.Exception_Handler(sys.exc_info(), None, "Could not load inspector extension")

try:
metaData = {
"testNo": CommonUtil.current_tc_no,
"testName": CommonUtil.current_tc_name,
Expand All @@ -475,7 +465,15 @@ def set_extension_variables():
"url": url,
"apiKey": apiKey,
"jwtKey": jwtKey,
"nodeId": Shared_Resources.Get_Shared_Variables('node_id'),
}
with open(Path(aiplugin_path) / "data.json", "w") as file:
json.dump(metaData, file, indent=4)

except:
return CommonUtil.Exception_Handler(sys.exc_info(), None, "Could not load inspector extension")

try:
with open(Path(ai_recorder_path) / "background" / "data.json", "w") as file:
json.dump(metaData, file, indent=4)
with open(Path(ai_recorder_public_path) / "background" / "data.json", "w") as file:
Expand Down
102 changes: 97 additions & 5 deletions Framework/MainDriverApi.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@
from settings import PROJECT_ROOT
from reporting import junit_report

from jinja2 import Environment, FileSystemLoader
from genson import SchemaBuilder
import selenium

from rich.style import Style
from rich.table import Table
from rich.console import Console
from rich.box import ASCII_DOUBLE_HEAD, DOUBLE
from rich.padding import Padding
from jinja2 import Environment, FileSystemLoader

rich_print = Console().print

top_path = os.path.dirname(os.getcwd())
Expand Down Expand Up @@ -797,7 +799,7 @@ def cleanup_driver_instances(): # cleans up driver(selenium, appium) instances
pass


def advanced_float(text):
def advanced_float(text:str) -> float:
try:
return float(text)
except:
Expand Down Expand Up @@ -960,7 +962,6 @@ def run_test_case(
):
try:
TestCaseStartTime = time.time()
shared.Set_Shared_Variables("run_id", run_id)
test_case = str(TestCaseID).replace("#", "no")
CommonUtil.current_tc_no = test_case
CommonUtil.current_tc_name = testcase_info['title']
Expand Down Expand Up @@ -1124,7 +1125,9 @@ def run_test_case(
except:
pass

if not CommonUtil.debug_status:
if CommonUtil.debug_status:
send_dom_variables()
else:
CommonUtil.Join_Thread_and_Return_Result("screenshot")
if str(shared.Get_Shared_Variables("zeuz_auto_teardown")).strip().lower() not in ("off", "no", "false", "disable"):
cleanup_driver_instances()
Expand Down Expand Up @@ -1176,6 +1179,94 @@ def run_test_case(
return "passed"


def send_dom_variables():
try:
sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME
variables = []
max_threshold = 30000
for var_name in shared.shared_variables:
var_value = shared.shared_variables[var_name]
try:
if len(json.dumps(var_value)) > max_threshold:
builder = SchemaBuilder()
builder.add_object(var_value)
variables.append({
"type": "json_schema",
"variable_name": var_name,
"variable_value": builder.to_schema(),
"description": "",
})
else:
variables.append({
"type": "json_object",
"variable_name": var_name,
"variable_value": var_value,
"description": "",
})
except (json.decoder.JSONDecodeError, TypeError):
# CommonUtil.Exception_Handler(sys.exc_info())
variables.append({
"type": f"non_json: {str(type(var_value))}",
"variable_name": var_name,
"variable_value": "",
"description": "",
})
except Exception:
CommonUtil.Exception_Handler(sys.exc_info())

if shared.Test_Shared_Variables('selenium_driver'):
try:
selenium_driver = shared.Get_Shared_Variables("selenium_driver")
dom = selenium_driver.execute_script("""
var html = document.createElement('html');
html.setAttribute('zeuz','aiplugin');
var myString = document.documentElement.outerHTML;
html.innerHTML = myString;

var elements = html.getElementsByTagName('head');
while (elements[0])
elements[0].parentNode.removeChild(elements[0])

var elements = html.getElementsByTagName('link');
while (elements[0])
elements[0].parentNode.removeChild(elements[0])

var elements = html.getElementsByTagName('script');
while (elements[0])
elements[0].parentNode.removeChild(elements[0])

var elements = html.getElementsByTagName('style');
while (elements[0])
elements[0].parentNode.removeChild(elements[0])

return html.outerHTML.replace(/\s+/g, ' ').replace(/>\s+</g, '><');""")
except selenium.common.exceptions.JavascriptException:
CommonUtil.Exception_Handler(sys.exc_info())
dom = ""
except:
dom = None
else:
dom = None

data = {
"variables": variables,
"dom_web": {"dom": dom},
"node_id": shared.Get_Shared_Variables('node_id')
}
res = RequestFormatter.request("post",
RequestFormatter.form_uri("node_ai_contents/"),
data=json.dumps(data),
verify=False
)
if res.status_code == 500:
CommonUtil.ExecLog(sModuleInfo, res.json()["info"], 3)
elif res.status_code == 404:
CommonUtil.ExecLog(sModuleInfo, 'The chatbot API does not exist, server upgrade needed', 2)
return
except:
CommonUtil.Exception_Handler(sys.exc_info())


def set_device_info_according_to_user_order(device_order, device_dict, test_case_no, test_case_name, user_info_object, Userid, **kwargs):
# Need to set device_info for browserstack here
global device_info
Expand Down Expand Up @@ -1778,6 +1869,7 @@ def main(device_dict, all_run_id_info):
shared.Set_Shared_Variables("zeuz_collect_browser_log", "on")

shared.Set_Shared_Variables("run_id", run_id)
shared.Set_Shared_Variables("node_id", CommonUtil.MachineInfo().getLocalUser())

send_log_file_only_for_fail = ConfigModule.get_config_value("RunDefinition", "upload_log_file_only_for_fail")
send_log_file_only_for_fail = False if send_log_file_only_for_fail.lower() == "false" else True
Expand Down
40 changes: 40 additions & 0 deletions Projects/selenium-chrome.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from selenium.webdriver.remote.webelement import WebElement
import selenium
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service


options = Options()
service = Service()
# options.add_argument("disable-features=SameSiteByDefaultCookies,CookiesWithoutSameSiteMustBeSecure")
# options.add_argument("profile.content_settings.exceptions.cookies.allowed_for_urls=www.hubspot.com")

selenium_driver = selenium.webdriver.Chrome(
service=service,
options=options,
)

# selenium_driver = selenium.webdriver.Edge(options=options)

d={'__pdst': '179f24cb92cc4b9fba62e6cac1ac195b', '__hs_cookie_cat_pref': '1:true_2:true_3:true', 'hubspotutk': '1a9e5de2eb8087cacdf0bb072717e2fe', 'trcksesh': 'a86fd7db-3f4b-4751-a21b-c159b45e4f49', 'hs_login_email': 'sweta.kumari@astegic.in', '_conv_r': 's%3Awww.google.com*m%3Aorganic*t%3A*c%3A', 'hubspotapi-prefs': '0', 'hubspotapi-csrf': 'AAccUfsBgQZiqnIRaBxrCzG9WJbisnCrwdyoVzM36tbsOTf7_NdzlsO14_oLW0NKlGOcZbTJ4xdZFqMCm7_cYDly9bLY0YM-ag', 'csrf.app': 'AAccUfsBgQZiqnIRaBxrCzG9WJbisnCrwdyoVzM36tbsOTf7_NdzlsO14_oLW0NKlGOcZbTJ4xdZFqMCm7_cYDly9bLY0YM-ag', '_gcl_au': '1.1.1170923742.1719316319', '__hssrc': '1', '_fbp': 'fb.1.1719316319317.513096624546995196', 'IR_gbd': 'hubspot.com', '_tt_enable_cookie': '1', '_ttp': 'ffGsb6Z01Bri0BUBEPWw85fCU1P', 'IR_PI': '3232476b-265b-11ef-8ded-8b65d6232029%7C1719316320172', '_gid': 'GA1.2.314381481.1719316321', 'saw_experiment': 'true', '_conv_v': 'vi%3A1*sc%3A5*cs%3A1719393996*fs%3A1719303070*pv%3A5*seg%3A%7B10031564.1-10034364.1-10034366.1%7D*exp%3A%7B100341862.%7Bv.1003162477-g.%7B100327900.1-100328009.1%7D%7D-100342007.%7Bv.1003162876-g.%7B100327900.1-100328009.1%7D%7D%7D*ps%3A1719387239', '_conv_s': 'si%3A5*sh%3A1719393996167-0.8504025071243075*pv%3A1', '__hstc': '20629287.1a9e5de2eb8087cacdf0bb072717e2fe.1719316319087.1719387239517.1719393997151.3', '__hssc': '20629287.1.1719393997151', '_rdt_uuid': '1719316319750.38ecc6ee-3653-4eba-8930-db70315c3c0e', '_uetsid': '7cdd73f032ca11efa5d22f6516591ab8', '_uetvid': 'b387ae002a2c11efbe2915d15a4d5b6f', 'IR_12893': '1719393998315%7C2643745%7C1719393998315%7C%7C', '_ga': 'GA1.1.1261872802.1719316321', '_ga_LXTM6CQ0XK': 'GS1.1.1719393998.3.1.1719393998.60.0.0'}
d2 = [{'domain': 'app.hubspot.com', 'expiry': 1727173657, 'httpOnly': False, 'name': 'NPS_e579c260_last_seen', 'path': '/', 'sameSite': 'Lax', 'secure': True, 'value': '1719397657624'}, {'domain': '.hubspot.com', 'expiry': 1750933655, 'httpOnly': False, 'name': 'csrf.app', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'AAccUfs4fvUUifmMiPSYV5qkMAR03Hga6xUh-mgbTjt3e6ARJX-5Rb5zWFdKARR_E_IsUUqH3b24ro5z2Ry8BIeMNDoiP6079Q'}, {'domain': '.hubspot.com', 'expiry': 1750933655, 'httpOnly': False, 'name': 'hubspotapi-csrf', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'AAccUfs4fvUUifmMiPSYV5qkMAR03Hga6xUh-mgbTjt3e6ARJX-5Rb5zWFdKARR_E_IsUUqH3b24ro5z2Ry8BIeMNDoiP6079Q'}, {'domain': '.hubspot.com', 'expiry': 1734949538, 'httpOnly': False, 'name': '__hs_cookie_cat_pref', 'path': '/', 'sameSite': 'Lax', 'secure': False, 'value': '1:true_2:true_3:true'}, {'domain': '.hubspot.com', 'expiry': 1750933655, 'httpOnly': False, 'name': 'hubspotapi-prefs', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': '0'}, {'domain': '.hubspot.com', 'expiry': 1734949655, 'httpOnly': True, 'name': 'hs_c2l', 'path': '/', 'sameSite': 'Lax', 'secure': True, 'value': 'GPvenE0iGQAQstEnSMKEJv1kiridxD-LIQWNV_DBXaw'}, {'domain': '.hubspot.com', 'httpOnly': True, 'name': 'hubspotapi', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'AAccUfv_C4SsS5AjhZIAT9Rxoc3Co2-9CwHyTpxaP1-7nNNCEJWoHEzCwMKAHMHFCgU8me4RJmA6yVwGoPrn8nrKU7oXU1irdXMcdtKnCUpk4PWfUvt008CkCf74TA18eiFRfX6N8joRV40mR5f8xDqTe3rMhyz33w5Hp_ZAY9uKStt0E4I9NAK4XrNy9ucrrPqLPGxajrtTmAVHS6pMneQxOSc7FLK4cr5tRbjvtCUgdAuxovG_KhIXFfq4GSgaGKyX71_C9SGvlQ8FX9wvOTxHdwZHPSGHZUEoGrcMaaR1cgRpdLXC1K1ZieHobl2PZ1BRAz6lMaMfpeCiAFN7-ZBpUvjs3d-4XfzHHDGOP61kQVLrSyZYZm87Si8Vhx2YuTd0LJcDVmoSbKiF5q5OB-9lT_8ADvUYsCaG8-TJh-h0P8xakafMsa1zHaYNhLbZ7SA2yzXwslTfQDov3zM4g1hWFFiDNmD1LbRaPfgaAGdTKGhFyfziz3sAqdMvXl8cBHFRkYHLUDenqLttg7ZymeH1BIgW45zmVCybLo5est7MTlKiQIe9NCa2ZVDQwqSBmf6HvdsYOVCeuEKwrVVsjf2tQx40GGuXIOhptvZ7FoEjCQZJgvaZl5p0OfkRYJW6CrbWy7r7BIOipzPBUcZJ0Y9jkBJe1TZTSPyqBMQzbNiyX-Q0hugQEwODw_EjO-zuyPR3ZCzhWws9YXs0A9WP8SE9aB3bMFPdU9fjXW8E0TLbgOq5SHYlPTB7AMeQH_RunFMC1hTjUGVvg-e2lEo3l_26IbTjSuw7xsSXAEoPpXwStKgTHtafe8ykwuKU3wG-P37YvgNU6Gj2CsRUVmmw9OqRUmQB7lGXcKhz0juY4jvU0qSTaFQRxFzeSlhNmEmiQ1j2P-A8mk8U9_Z8DTP5y9K6np3nqiMAvPEkMIGo39Evykzrtun9KjGC9f9AFXKL4fG6HWfAs_ESIv0NBRNOAsJCVIqKwRxFsQDRltR_chwlI8MX7gcrWehZQIR_4r2hnFkzKJu9LgmPMSWZ2Vi9kTcgm_PaWVXmgpVDOqBgjwiwN36JzHrjq8LI569QR72IVoeoG31mbngzhFOGv4SKYIQOGPBbLUw8UwjiOX9LXPgm4TIDMYfSdk8pnDmJjbkbBChjmohJilF3Us2RzDmE_NLzZYRvr6Sb4FdvdmR0Mq2N16RHRsDVEznVd4HFZp4WpO1zrH-qUS9kGh6Wu1flOygqrHdC3xCggA'}, {'domain': '.hubspot.com', 'httpOnly': True, 'name': '_cfuvid', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'VHVahDU40Dmu.1Qz980FJSPww1NhiLWl0Vccm1nqHeI-1719397525545-0.0.1.1-604800000'}, {'domain': '.hubspot.com', 'expiry': 1750933655, 'httpOnly': False, 'name': 'hs_login_email', 'path': '/', 'sameSite': 'Lax', 'secure': True, 'value': 'sweta.kumari@astegic.in'}, {'domain': '.hubspot.com', 'expiry': 1719399324, 'httpOnly': True, 'name': '__cf_bm', 'path': '/', 'sameSite': 'None', 'secure': True, 'value': 'DZACl4TpMi4DvonGPzq7smssgHwLiW0P6qv8_wG6sKY-1719397525-1.0.1.1-vpGOK0sWRMpGBwZQ_Sih0UFgwJ92YnBi_PDcXRqolS.uypW0iMyS7XTMchxyEFJQ758x1vLoIMmd6tZGxryH5Q'}]
d3 = [
{
'name': '_cfuvid',
'value': '6q43VvZCGxj5SYyd6P0jD6IJ.uNBlgwO9dZAsTB0EBU-1719398293577-0.0.1.1-604800000',
'domain': '.hubspot.com'
}
]
selenium_driver.get('https://app.hubspot.com/user-guide/46427382?via=home')

print('Before:', selenium_driver.get_cookies())
for each in d2:
try:
selenium_driver.add_cookie(each)
except Exception as e: print(e)
print('After:', selenium_driver.get_cookies())
selenium_driver.get('https://app.hubspot.com/user-guide/46427382?via=home')

print()
input('teardown?')
selenium_driver.quit()
1 change: 1 addition & 0 deletions requirements-linux.txt
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,5 @@ pandas
pyperclip
thefuzz
backports-datetime-fromisoformat; python_version < '3.11'
genson
google-cloud-storage
1 change: 1 addition & 0 deletions requirements-mac.txt
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,5 @@ pandas
pyperclip
backports-datetime-fromisoformat; python_version < '3.11'
thefuzz
genson
google-cloud-storage
Loading