Improved variable detection

deepsecrets/core/tokenizers/helpers/semantic/var_detection/rules.py

@@ -102,7 +102,11 @@ class VariableDetectionRules:
         VaribleDetector(
             language=Language.ANY,
             stream_pattern=re.compile('(v|n)(p|o)(L)'),
-            match_rules={2: Match(values=[':', '='])},
+            match_rules={
+                2: Match(values=[
+                    re.compile('^:$'),
+                    re.compile('^=$'),
+                ])},
             match_semantics={1: 'name', 3: 'value'},
         ),
         VaribleDetector(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepsecrets"
-version = "1.0.5"
+version = "1.0.6"
 description = "A better tool for secrets search"
 license = "MIT"
 authors = [

tests/core/engines/semantic/test_semantic.py

@@ -33,6 +33,11 @@ def file_sh_2() -> File:
     path = 'tests/fixtures/2.sh'
     return File(path=path, relative_path=path)
 
+@pytest.fixture(scope='module')
+def file_html_1() -> File:
+    path = 'tests/fixtures/1.html'
+    return File(path=path, relative_path=path)
+
 
 def test_1_semantic_engine(file: File):
     tokens = LexerTokenizer(deep_token_inspection=True).tokenize(file)
@@ -120,4 +125,23 @@ def test_5_semantic_engine(file_sh_2: File):
 
     findings = FindingMerger(findings).merge()
     assert len(findings) == 1
-    assert findings[0].final_rule.name == 'Dangerous condition'
+    assert findings[0].final_rule.name == 'Dangerous condition'
+
+
+def test_6_semantic_engine(file_html_1: File):
+    tokens = LexerTokenizer(deep_token_inspection=True).tokenize(file_html_1)
+    #assert len(tokens) == 16
+
+    engine = SemanticEngine(subengine=None)
+
+    findings: List[Finding] = []
+    for token in tokens:
+        findings.extend(engine.search(token))
+
+    for finding in findings:
+        finding.map_on_file(file=file_html_1, relative_start=finding.start_pos)
+        finding.choose_final_rule()
+
+
+    findings = FindingMerger(findings).merge()
+    assert len(findings) == 0

tests/core/tokenizers/lexer/variable_detection/test_html.py

@@ -0,0 +1,18 @@
+import pytest
+
+from deepsecrets.core.model.file import File
+from deepsecrets.core.tokenizers.lexer import LexerTokenizer
+
+
+@pytest.fixture(scope='module')
+def file_html_1():
+    path = 'tests/fixtures/1.html'
+    return File(path=path, relative_path=path)
+
+
+def test_1(file_html_1):
+    lex = LexerTokenizer(deep_token_inspection=True)
+    lex.tokenize(file_html_1, post_filter=False)
+
+    variables = lex.get_variables()
+    assert len(variables) == 32

tests/fixtures/1.html

@@ -0,0 +1,109 @@
+{% extends 'labels/base.html' %}
+{% load static %}
+
+{% block contentcss %}
+    <link rel="stylesheet" href="{% static "labels/css/labeling.task.css" %}">
+{% endblock %}
+
+{% block content %}
+    <div class="content">
+        {% if session and not user.is_authenticated %}
+            <span id="session">session: {{ session }}</span>
+        {% endif %}
+        <div class="container">
+            <div class="row justify-content-center">
+                <div class="card col-9 mt-2">
+                    <div class="card-body">
+                        <h5 class="card-title"><span style="font-weight: normal">Запрос </span>
+                            <a href="https://example.com?q={{ task.query_text }}" target="labeling-queries">
+                                <span>{{ task.query_text }}</span>
+                            </a></h5>
+                        <hr/>
+                        <h5 class="card-title"><a href="https://example.com/{{ task.item_id }}" target="labeling-items">
+                            <span>{{ task.item_title }}</span>
+                        </a></h5>
+                        <h6 class="card-subtitle mb-2 text-muted">{{ task.item_category }}</h6>
+                        {% if task.item_img %}
+                            <img style="display: block; margin: 0 auto;" src="{{ task.item_img }}" alt="item image"/>
+                        {% endif %}
+                        <div class="card-text" style="white-space: pre-line">{{ task.item_description|safe }}</div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+    <footer class="footer">
+        <div id="submission">
+            <button class="btn btn-success" data-label="2">Подходит</button>
+            <button class="btn btn-secondary" data-label="1">Непонятно</button>
+            <button class="btn btn-danger" data-label="0">Не подходит</button>
+        </div>
+    </footer>
+{% endblock %}
+
+{% block contentjs %}
+    <script>
+        $('#submission').find('button').each(function () {
+            $(this).click(function () {
+                $(this).attr('disabled', 'disabled');
+                {% autoescape off %}
+                    var url = "{% url 'labeling_task_submit' %}";
+                {% endautoescape %}
+                var label = $(this).data('label');
+                var params = {
+                    "project_id": {{ project_id }},
+                    "query_id": {{ task.query_id }},
+                    "item_id": {{ task.item_id }},
+                    "label": label
+                };
+                $.post(url, params, function (data, status) {
+                    ShowNext()
+                });
+            })
+        });
+
+        function ShowNext() {
+            {% autoescape off %}
+                var url = "{% url 'labeling_task' %}?project_id={{ project_id }}";
+            {% endautoescape %}
+            {% if steps_back %}
+                var steps_back = {{ steps_back }};
+                url = url + '&steps_back=' + (steps_back - 1);
+            {% endif %}
+            window.location.href = url;
+        }
+
+        function ShowPrevious() {
+            {% autoescape off %}
+                var url = "{% url 'labeling_task' %}?project_id={{ project_id }}";
+            {% endautoescape %}
+            var steps_back = 0;
+            {% if steps_back %}
+                steps_back = {{ steps_back }};
+            {% endif %}
+            url = url + '&steps_back=' + (steps_back + 1);
+            window.location.href = url;
+        }
+
+        $(window).keydown(function (e) {
+            if (e.key === '1') {
+                $('#submission .btn-success').click();
+            } else if (e.key === '2') {
+                $('#submission .btn-secondary').click();
+            } else if (e.key === '3') {
+                $('#submission .btn-danger').click();
+            } else if (e.key === ',' && e.metaKey && e.shiftKey) {
+                ShowPrevious();
+            } else if (e.key === 'ArrowLeft' && e.ctrlKey && e.shiftKey) {
+                ShowPrevious();
+            } else if (e.key === '.' && e.metaKey && e.shiftKey) {
+                ShowNext();
+            } else if (e.key === 'ArrowRight' && e.ctrlKey && e.shiftKey) {
+                ShowNext();
+            } else {
+                return
+            }
+            $(window).off('keydown');
+        });
+    </script>
+{% endblock %}