Merge pull request #1383 from yogeshojha/1356-feat-store-scan-related…

…-configuration-such-as-imported-subdomains-out-of-scope-subdomains-starting-point-url-and-ignore-paths

feat: Store and showcase scan related configuration such as imported subdomains, out of scope subdomains, starting point url and excluded paths fixes #1356

web/reNgine/celery_custom_task.py

@@ -67,7 +67,7 @@ def __call__(self, *args, **kwargs):
 		self.subscan_id = ctx.get('subscan_id')
 		self.engine_id = ctx.get('engine_id')
 		self.filename = ctx.get('filename')
-		self.starting_point_url = ctx.get('starting_point_url', '')
+		self.starting_point_path = ctx.get('starting_point_path', '')
 		self.excluded_paths = ctx.get('excluded_paths', [])
 		self.results_dir = ctx.get('results_dir', RENGINE_RESULTS)
 		self.yaml_configuration = ctx.get('yaml_configuration', {})

web/reNgine/tasks.py

@@ -57,7 +57,7 @@ def initiate_scan(
 		imported_subdomains=[],
 		out_of_scope_subdomains=[],
 		initiated_by_id=None,
-		starting_point_url='',
+		starting_point_path='',
 		excluded_paths=[],
 	):
 	"""Initiate a new scan.
@@ -70,7 +70,7 @@ def initiate_scan(
 		results_dir (str): Results directory.
 		imported_subdomains (list): Imported subdomains.
 		out_of_scope_subdomains (list): Out-of-scope subdomains.
-		starting_point_url (str): URL path. Default: '' Defined where to start the scan.
+		starting_point_path (str): URL path. Default: '' Defined where to start the scan.
 		initiated_by (int): User ID initiating the scan.
 		excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
 	"""
@@ -92,7 +92,7 @@ def initiate_scan(
 		domain.save()
 
 		# Get path filter
-		starting_point_url = starting_point_url.rstrip('/')
+		starting_point_path = starting_point_path.rstrip('/')
 
 		# for live scan scan history id is passed as scan_history_id 
 		# and no need to create scan_history object
@@ -114,6 +114,12 @@ def initiate_scan(
 		scan.tasks = engine.tasks
 		scan.results_dir = f'{results_dir}/{domain.name}_{scan.id}'
 		add_gf_patterns = gf_patterns and 'fetch_url' in engine.tasks
+		# add configs to scan object, cfg_ prefix is used to avoid conflicts with other scan object fields
+		scan.cfg_starting_point_path = starting_point_path
+		scan.cfg_excluded_paths = excluded_paths
+		scan.cfg_out_of_scope_subdomains = out_of_scope_subdomains
+		scan.cfg_imported_subdomains = imported_subdomains
+
 		if add_gf_patterns:
 			scan.used_gf_patterns = ','.join(gf_patterns)
 		scan.save()
@@ -127,7 +133,7 @@ def initiate_scan(
 			'engine_id': engine_id,
 			'domain_id': domain.id,
 			'results_dir': scan.results_dir,
-			'starting_point_url': starting_point_url,
+			'starting_point_path': starting_point_path,
 			'excluded_paths': excluded_paths,
 			'yaml_configuration': config,
 			'out_of_scope_subdomains': out_of_scope_subdomains
@@ -152,7 +158,7 @@ def initiate_scan(
 
 		# If enable_http_crawl is set, create an initial root HTTP endpoint so that
 		# HTTP crawling can start somewhere
-		http_url = f'{domain.name}{starting_point_url}' if starting_point_url else domain.name
+		http_url = f'{domain.name}{starting_point_path}' if starting_point_path else domain.name
 		endpoint, _ = save_endpoint(
 			http_url,
 			ctx=ctx,
@@ -228,7 +234,7 @@ def initiate_subscan(
 		engine_id=None,
 		scan_type=None,
 		results_dir=RENGINE_RESULTS,
-		starting_point_url='',
+		starting_point_path='',
 		excluded_paths=[],
 	):
 	"""Initiate a new subscan.
@@ -239,7 +245,7 @@ def initiate_subscan(
 		engine_id (int): Engine ID.
 		scan_type (int): Scan type (periodic, live).
 		results_dir (str): Results directory.
-		starting_point_url (str): URL path. Default: ''
+		starting_point_path (str): URL path. Default: ''
 		excluded_paths (list): Excluded paths. Default: [], url paths to exclude from scan.
 	"""
 
@@ -298,13 +304,13 @@ def initiate_subscan(
 		'subdomain_id': subdomain.id,
 		'yaml_configuration': config,
 		'results_dir': results_dir,
-		'starting_point_url': starting_point_url,
+		'starting_point_path': starting_point_path,
 		'excluded_paths': excluded_paths,
 	}
 
 	# Create initial endpoints in DB: find domain HTTP endpoint so that HTTP
 	# crawling can start somewhere
-	base_url = f'{subdomain.name}{starting_point_url}' if starting_point_url else subdomain.name
+	base_url = f'{subdomain.name}{starting_point_path}' if starting_point_path else subdomain.name
 	endpoint, _ = save_endpoint(
 		base_url,
 		crawl=enable_http_crawl,
@@ -406,8 +412,8 @@ def subdomain_discovery(
 	if not host:
 		host = self.subdomain.name if self.subdomain else self.domain.name
 
-	if self.starting_point_url:
-		logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_url}).')
+	if self.starting_point_path:
+		logger.warning(f'Ignoring subdomains scan as an URL path filter was passed ({self.starting_point_path}).')
 		return
 
 	# Config
@@ -1930,7 +1936,7 @@ def fetch_url(self, urls=[], ctx={}, description=None):
 
 		if base_url and urlpath:
 			subdomain = urlparse(base_url)
-			url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_url}'
+			url = f'{subdomain.scheme}://{subdomain.netloc}{self.starting_point_path}'
 
 		if not validators.url(url):
 			logger.warning(f'Invalid URL "{url}". Skipping.')
@@ -1939,8 +1945,8 @@ def fetch_url(self, urls=[], ctx={}, description=None):
 			all_urls.append(url)
 
 	# Filter out URLs if a path filter was passed
-	if self.starting_point_url:
-		all_urls = [url for url in all_urls if self.starting_point_url in url]
+	if self.starting_point_path:
+		all_urls = [url for url in all_urls if self.starting_point_path in url]
 
 	# if exclude_paths is found, then remove urls matching those paths
 	if self.excluded_paths:
@@ -2840,8 +2846,8 @@ def http_crawl(
 	input_path = f'{self.results_dir}/httpx_input.txt'
 	history_file = f'{self.results_dir}/commands.txt'
 	if urls: # direct passing URLs to check
-		if self.starting_point_url:
-			urls = [u for u in urls if self.starting_point_url in u]
+		if self.starting_point_path:
+			urls = [u for u in urls if self.starting_point_path in u]
 
 		with open(input_path, 'w') as f:
 			f.write('\n'.join(urls))

web/startScan/migrations/0002_auto_20240821_1518.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.23 on 2024-08-21 15:18
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('startScan', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scanhistory',
+            name='cfg_excluded_paths',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='scanhistory',
+            name='cfg_out_of_scope_subdomains',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='scanhistory',
+            name='cfg_starting_point_url',
+            field=models.CharField(blank=True, max_length=200, null=True),
+        ),
+    ]

web/startScan/migrations/0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-08-21 15:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('startScan', '0002_auto_20240821_1518'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='scanhistory',
+            old_name='cfg_starting_point_url',
+            new_name='cfg_starting_point_path',
+        ),
+    ]

web/startScan/migrations/0004_scanhistory_cfg_imported_subdomains.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.23 on 2024-08-22 04:26
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('startScan', '0003_rename_cfg_starting_point_url_scanhistory_cfg_starting_point_path'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scanhistory',
+            name='cfg_imported_subdomains',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=200), blank=True, default=list, null=True, size=None),
+        ),
+    ]

web/startScan/models.py

@@ -48,6 +48,26 @@ class ScanHistory(models.Model):
 	dorks = models.ManyToManyField('Dork', related_name='dorks', blank=True)
 	initiated_by = models.ForeignKey(User, on_delete=models.CASCADE, related_name='initiated_scans', blank=True, null=True)
 	aborted_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True, related_name='aborted_scans')
+	# scan related configs, prefix config fields with cfg_
+	cfg_out_of_scope_subdomains = ArrayField(
+		models.CharField(max_length=200),
+		blank=True,
+		null=True,
+		default=list
+	)
+	cfg_starting_point_path = models.CharField(max_length=200, blank=True, null=True)
+	cfg_excluded_paths = ArrayField(
+		models.CharField(max_length=200),
+		blank=True,
+		null=True,
+		default=list
+	)
+	cfg_imported_subdomains = ArrayField(
+		models.CharField(max_length=200),
+		blank=True,
+		null=True,
+		default=list
+	)
 
 
 	def __str__(self):

web/startScan/static/startScan/js/detail_scan.js

@@ -1039,12 +1039,12 @@ function create_log_element(log) {
 	let logElement = document.createElement("p");
 	innerHTML = `
 	<p>
-	  <p data-bs-toggle="collapse" data-bs-target="#collapse${log.id}">
-		<b>${log.command}</b>
+	  <p data-bs-toggle="collapse" data-bs-target="#collapse${log.id}" class="text-primary">
+		<i class="fe-terminal"></i>${log.command}
 	  </p>
 	</p>`
 	if (log.output != ''){
-		innerHTML += `<div class="collapse" id="collapse${log.id}"><div style="white-space: pre-line" class="card card-body">${log.output}</div></div>`;
+		innerHTML += `<div class="collapse" id="collapse${log.id}"><code style="white-space: pre-line" class="card card-body">${log.output}</code></div>`;
 	}
 	logElement.innerHTML = innerHTML;
 	return logElement;

web/startScan/templates/startScan/_items/schedule_scan_wizard.html

@@ -89,8 +89,8 @@ <h4 class="text-warning">Out of Scope Subdomains (Optional)</h4>
                         <h3>URL Scope and Exclusions</h3>
                         <div class="mb-4">
                             <div class="mb-3">
-                                <h4 class="text-info">Starting Point URL (Optional)</h4>
-                                <input type="email" class="form-control" id="startingPointUrl" placeholder="e.g. /home" name="startingPointUrl">
+                                <h4 class="text-info">Starting Point Path (Optional)</h4>
+                                <input type="email" class="form-control" id="startingPointPath" placeholder="e.g. /home" name="startingPointPath">
                                 <small class="form-text text-muted">
                     Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains.
                     <br>

web/startScan/templates/startScan/_items/start_scan_wizard.html

@@ -52,8 +52,8 @@ <h4 class="text-warning">Out of Scope Subdomains (Optional)</h4>
             <h4>URL Scope and Exclusions</h4>
             <div class="mb-4">
               <div class="mb-3">
-                <h4 class="text-info">Starting Point URL (Optional)</h4>
-                <input type="email" class="form-control" id="startingPointUrl" placeholder="e.g. /home" name="startingPointUrl">
+                <h4 class="text-info">Starting Point Path (Optional)</h4>
+                <input type="email" class="form-control" id="startingPointPath" placeholder="e.g. /home" name="startingPointPath">
                 <small class="form-text text-muted">
                   Defines where the scan should begin. Leave blank to scan from the root (/) and include all subdomains. 
                   </br>

web/startScan/templates/startScan/detail_scan.html

@@ -146,11 +146,51 @@ <h5 class="mt-0 mb-0 font-15">
 							<p class="text-danger">Scan failed due to ERROR: {{history.error_message}}</p>
 						{% endif %}
 						{% endif %}
-						<h6 class="font-13 text-muted text-uppercase">Scan Logs</h6>
-						<span><a href="javascript:get_logs_modal({{history.id}})"><i class="fe-file"></i> Logs</a></span>
-						<h6 class="font-13 text-muted text-uppercase">Scan Engine</h6>
+						<h6 class="font-13 text-muted text-uppercase"><i class="fe-settings"></i> Scan Configurations</h6>
+						<span>Starting Path: {% if history.cfg_starting_point_path %} {{history.cfg_starting_point_path}} {% else %}/ {% endif %}</span><i class="fa fa-info-circle text-muted float-end" data-bs-container="#tooltip-container" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Scan path where the scan began"></i><br>
+						<div class="accordion custom-accordion" id="custom-accordion-one">
+							<div>
+								<a class="custom-accordion-title text-reset d-block collapsed" data-bs-toggle="collapse" href="#collapseImported" aria-expanded="true" aria-controls="collapseImported">
+									{{history.cfg_imported_subdomains|length}} Imported Subdomains <i class="mdi mdi-chevron-down accordion-arrow"></i>
+								</a>
+								<div id="collapseImported" class="collapse" data-bs-parent="#custom-accordion-one" style="">
+									<ul>
+										{% for subdomain in history.cfg_imported_subdomains %}
+										<li><code>{{subdomain}}</code></li>
+										{% endfor %}
+									</ul>
+								</div>
+							</div>
+							<div>
+								<a class="custom-accordion-title text-reset d-block collapsed" data-bs-toggle="collapse" href="#collapseOutofScope" aria-expanded="true" aria-controls="collapseImported">
+									{{history.cfg_out_of_scope_subdomains|length}} out of scope Subdomains <i class="mdi mdi-chevron-down accordion-arrow"></i>
+								</a>
+								<div id="collapseOutofScope" class="collapse" data-bs-parent="#custom-accordion-one" style="">
+									<ul>
+										{% for subdomain in history.cfg_out_of_scope_subdomains %}
+										<li><code>{{subdomain}}</code></li>
+										{% endfor %}
+									</ul>
+								</div>
+							</div>
+							<div>
+								<a class="custom-accordion-title text-reset d-block collapsed" data-bs-toggle="collapse" href="#collapsePath" aria-expanded="true" aria-controls="collapseImported">
+									{{history.cfg_excluded_paths|length}} Excluded Paths <i class="mdi mdi-chevron-down accordion-arrow"></i>
+								</a>
+								<div id="collapsePath" class="collapse" data-bs-parent="#custom-accordion-one" style="">
+									<ul>
+										{% for path in history.cfg_excluded_paths %}
+										<li><code>{{path}}</code></li>
+										{% endfor %}
+									</ul>
+								</div>
+							</div>
+						</div>
+						<h6 class="font-13 text-muted text-uppercase mt-3"><i class="fe-file"></i> Scan Logs</h6>
+						<span><a href="javascript:get_logs_modal({{history.id}})"><i class="fe-terminal"></i> Logs</a></span>
+						<h6 class="font-13 text-muted text-uppercase mt-3"><i class="fe-cpu"></i> Scan Engine</h6>
 						<span class="badge badge-soft-primary">{{history.scan_type.engine_name}}</span>
-						<h6 class="font-13 text-muted text-uppercase mt-3">Scan Duration </h6>
+						<h6 class="font-13 text-muted text-uppercase mt-3"><i class="fe-clock"></i> Scan Duration </h6>
 						{% if history.scan_status == -1 %}
 						<p class="mt-1 mb-0 text-warning font-14"><small class="text-center text-muted">Scan not yet started.</small></p>
 						{% elif history.scan_status == 0 %}
@@ -166,9 +206,7 @@ <h6 class="font-13 text-muted text-uppercase mt-3">Scan Duration </h6>
 						{% elif history.scan_status == 3 %}
 						<span class="mt-1 mb-0 text-danger font-14">Aborted in {{ history.start_scan_date|timesince:history.stop_scan_date }}</span>
 						{% endif %}
-
-
-						<h3 class="font-13 text-muted text-uppercase mb-2">Scan Progress</h3>
+						<h3 class="font-13 text-muted text-uppercase mt-3"><i class="fe-loader"></i> Scan Progress</h3>
 						<div class="progress mb-2 progress-md">
 							{% if history.scan_status == -1 %}
 							<div class="progress-bar bg-warning" role="progressbar" style="width: 10%" aria-valuemin="0" aria-valuemax="100">
@@ -200,7 +238,7 @@ <h3 class="font-13 text-muted text-uppercase mb-2">Tagged to Organization</h3>
 						{% endfor %}
 						{% endif %}
 
-						<h6 class="font-13 text-muted text-uppercase mt-4 mb-2">Scan Timeline</h6>
+						<h6 class="font-13 text-muted text-uppercase mt-4 mb-2"><i class="fe-activity"></i> Timeline</h6>
 						<div data-simplebar style="max-height: 410px;" class="p-2">
 							<div class="track-order-list">
 								<ul class="list-unstyled">
@@ -220,7 +258,7 @@ <h5 class="mt-0 mb-1">{{activity.title}}
 												<p class="badge badge-soft-danger">Error: {{activity.error_message}}</p>
 											{% endif %}
 										{% endif %}
-										<span><a href="javascript:get_logs_modal(null, {{activity.id}})"><i class="fe-file"></i> Logs</a></span>
+										<span><a href="javascript:get_logs_modal(null, {{activity.id}})"><i class="fe-terminal"></i> Logs</a></span>
 									</li>
 									{% endfor %}
 								</ul>

web/startScan/templates/startScan/history.html

@@ -97,6 +97,10 @@ <h4 class="headline-title">Filters</h4>
               <td class="">
                 {{ scan_history.domain.name }}
                 <br>
+                {% if scan_history.cfg_starting_point_path %}
+                  <span><i class=""></i>Path: <code>{{scan_history.cfg_starting_point_path}}</code></span>
+                {% endif %}
+                <br>
                 {% for organization in scan_history.domain.get_organization %}
                 <span class="badge badge-soft-dark mt-1 me-1" data-toggle="tooltip" data-placement="top" title="Domain {{domain.name}} belongs to organization {{organization.name}}">{{ organization.name }}</span>
                 {% endfor %}
@@ -173,6 +177,7 @@ <h4 class="headline-title">Filters</h4>
                         <i class="mdi mdi-chevron-right"></i>
                       </button>
                       <div class="dropdown-menu" style="">
+                        <a class="dropdown-item text-primary" href="javascript:show_scan_configuration('{{scan_history.cfg_starting_point_path}}', {{scan_history.cfg_out_of_scope_subdomains}}, {{scan_history.cfg_excluded_paths}}, {{scan_history.cfg_imported_subdomains}})"><i class="fe-settings"></i>&nbsp;Show Configurations</a>
                         {% if user|can:'initiate_scans_subscans' %}
                           {% if scan_history.scan_status == 0 or scan_history.scan_status == 2 or scan_history.scan_status == 3 %}
                           <a class="dropdown-item text-primary" href="/scan/{{current_project.slug}}/start/{{scan_history.domain.id}}">