From 8d3fdc3380ee3d0b3f145d47020c62ca8a096120 Mon Sep 17 00:00:00 2001 From: ScriptTiger Date: Mon, 16 Sep 2019 09:45:19 +0800 Subject: [PATCH] Format refinements Follow-up to the following issues: https://github.com/ScriptTiger/Hosts-Conversions/issues/1 https://github.com/ScriptTiger/scripttiger.github.io/issues/2 --- DNSmasq.cmd | 90 +++++++++++++++++++++++++++++++++++++++++++--------- Unbound.cmd | 91 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 150 insertions(+), 31 deletions(-) diff --git a/DNSmasq.cmd b/DNSmasq.cmd index 1508084..850411b 100644 --- a/DNSmasq.cmd +++ b/DNSmasq.cmd @@ -22,32 +22,92 @@ rem Read the below section carefully and adjust any variables as neeeded rem Blackhole address for source file set FROM_BLACKHOLE=0.0.0.0 -rem Blackhole address for resultant file -set TO_BLACKHOLE=0.0.0.0 - rem 0 to remove commments, 1 to keep them set COMMENTS=0 rem ===== -echo Converting "%~1" to "%~dp0DNSmasq-%~nx1"... +rem Only adjust the following variables if you know what you're doing +set CACHE=Hosts-Conversions +set CACHE=%TEMP%\!CACHE! +set CTEMP=!CACHE!\ctemp -rem Capture all output to a single write operation data stream -rem Forcing a write operating each line is considerably slower when dealing with files of higher line counts +if not exist "!CACHE!" md "!CACHE!" + +rem Generate an index file of only domain names for faster cross-checking when removing subdomains +echo Generating domain index from "%~1"... +( + rem List only domain names and enforce clean input + for /f "tokens=1,2*" %%a in ( + 'findstr /b "!FROM_BLACKHOLE:.=[.]!" "%~s1" ^| findstr /b /v /c:"0.0.0.0 0.0.0.0"' + ) do echo %%b +) > "!CTEMP!" + +rem Cross-check all possible parent domains from each entry in the hosts file with exact matches from the index +rem If the entry is a subdomain of a parent domain already indexed, don't include the subdomain +echo Converting "%~1" to "%~dp0DNSmasq-%~nx1"... ( - rem Read from the source hosts file and enfore clean input + rem Enforce clean input from only comment lines and domain name lines, everything else is removed for /f "tokens=1,2*" %%a in ( 'findstr /b "!FROM_BLACKHOLE:.=[.]! #" "%~s1" ^| findstr /b /v /c:"0.0.0.0 0.0.0.0"' ) do ( - if "%%a"=="!FROM_BLACKHOLE!" ( - echo address=/%%b/!TO_BLACKHOLE! - ) else ( - if !COMMENTS!==1 ( - set LINE=%%a %%b %%c - if "!LINE:~,1!"=="#" echo !LINE! + set LINE=%%a %%b %%c + rem If the line is not a comment line, generate search strings for each parent domain to cross-check index + rem If the line is a comment line and comments are enabled, the leading "#" is converted to a ";" and the comment is written to the file + if not "!LINE:~,1!"=="#" ( + for /f "tokens=2,3,4,5,6,7,8,9* delims=." %%j in ("%%b") do ( + rem If the domain name is only a second-level domain, automatically write it and skip searching anything + if not "%%k"=="" ( + rem Rapidly generate search strings for all possible parent domains using as much known static data as possible to speed up the process + set FINDSTR= + if not "%%k"=="" set FINDSTR=%%j.%%k + if not "%%l"=="" set FINDSTR=%%j.%%k.%%l %%k.%%l + if not "%%m"=="" set FINDSTR=%%j.%%k.%%l.%%m %%k.%%l.%%m %%l.%%m + if not "%%n"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n %%k.%%l.%%m.%%n %%l.%%m.%%n %%m.%%n + if not "%%o"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o %%k.%%l.%%m.%%n.%%o %%l.%%m.%%n.%%o %%m.%%n.%%o %%n.%%o + if not "%%p"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o.%%p %%k.%%l.%%m.%%n.%%o.%%p %%l.%%m.%%n.%%o.%%p %%m.%%n.%%o.%%p %%n.%%o.%%p %%o.%%p + if not "%%q"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o.%%p.%%q %%k.%%l.%%m.%%n.%%o.%%p.%%q %%l.%%m.%%n.%%o.%%p.%%q %%m.%%n.%%o.%%p.%%q %%n.%%o.%%p.%%q %%o.%%p.%%q %%p.%%q + rem If the domain contains 9 parent domains or more, build the search string dynamically + if not "%%r"=="" ( + set PARENTS=%%r + set FINDSTR=%%j %%k %%l %%m %%n %%o %%p %%q !PARENTS:.= ! + set PARENTS= + for %%0 in (!FINDSTR!) do set PARENTS=%%0 !PARENTS! + call :Parents !PARENTS! + ) + rem Cross-check the index for any of the possible parent domains + rem Although using /x for exact match is dramatically slower than partial matches, it's a necessity + rem Phishing domains often stack domain names like XXX.com.YYY.com, and this could create problems for partial matches + findstr /l /x /m "!FINDSTR!" "!CTEMP!" > nul + rem If no parent domains are found in the index, write the domain to file + if !errorlevel!==1 echo address=/%%b/ + ) else echo address=/%%b/ ) - ) + ) else if !COMMENTS!==1 echo !LINE! ) ) > "%~dp0DNSmasq-%~nx1" + +rem Remove cache files +if exist "!CACHE!" ( + echo Cleaning temporary files... + rmdir /s /q "!CACHE!" +) + echo "%1" converted to "%~dp0DNSmasq-%~nx1" -pause \ No newline at end of file + +pause + +exit /b + +rem Function for handling domains with 9 parents or more +:Parents +set PARENTS=%2.%1 +set FINDSTR=%2.%1 +shift +shift +:Parents2 +shift +set PARENTS=%0.!PARENTS! +set FINDSTR=!PARENTS! !FINDSTR! +if not "%1"=="" goto Parents2 +exit /b \ No newline at end of file diff --git a/Unbound.cmd b/Unbound.cmd index a813494..5708bf1 100644 --- a/Unbound.cmd +++ b/Unbound.cmd @@ -22,33 +22,92 @@ rem Read the below section carefully and adjust any variables as neeeded rem Blackhole address for source file set FROM_BLACKHOLE=0.0.0.0 -rem Blackhole address for resultant file -set TO_BLACKHOLE=0.0.0.0 - rem 0 to remove commments, 1 to keep them set COMMENTS=0 rem ===== -echo Converting "%~1" to "%~dp0Unbound-%~nx1"... +rem Only adjust the following variables if you know what you're doing +set CACHE=Hosts-Conversions +set CACHE=%TEMP%\!CACHE! +set CTEMP=!CACHE!\ctemp -rem Capture all output to a single write operation data stream -rem Forcing a write operating each line is considerably slower when dealing with files of higher line counts +if not exist "!CACHE!" md "!CACHE!" + +rem Generate an index file of only domain names for faster cross-checking when removing subdomains +echo Generating domain index from "%~1"... +( + rem List only domain names and enforce clean input + for /f "tokens=1,2*" %%a in ( + 'findstr /b "!FROM_BLACKHOLE:.=[.]!" "%~s1" ^| findstr /b /v /c:"0.0.0.0 0.0.0.0"' + ) do echo %%b +) > "!CTEMP!" + +rem Cross-check all possible parent domains from each entry in the hosts file with exact matches from the index +rem If the entry is a subdomain of a parent domain already indexed, don't include the subdomain +echo Converting "%~1" to "%~dp0Unbound-%~nx1"... ( - rem Read from the source hosts file and enfore clean input + rem Enforce clean input from only comment lines and domain name lines, everything else is removed for /f "tokens=1,2*" %%a in ( 'findstr /b "!FROM_BLACKHOLE:.=[.]! #" "%~s1" ^| findstr /b /v /c:"0.0.0.0 0.0.0.0"' ) do ( - if "%%a"=="!FROM_BLACKHOLE!" ( - echo local-zone: "%%b" redirect - echo local-data: "%%b A !TO_BLACKHOLE!" - ) else ( - if !COMMENTS!==1 ( - set LINE=%%a %%b %%c - if "!LINE:~,1!"=="#" echo !LINE! + set LINE=%%a %%b %%c + rem If the line is not a comment line, generate search strings for each parent domain to cross-check index + rem If the line is a comment line and comments are enabled, the leading "#" is converted to a ";" and the comment is written to the file + if not "!LINE:~,1!"=="#" ( + for /f "tokens=2,3,4,5,6,7,8,9* delims=." %%j in ("%%b") do ( + rem If the domain name is only a second-level domain, automatically write it and skip searching anything + if not "%%k"=="" ( + rem Rapidly generate search strings for all possible parent domains using as much known static data as possible to speed up the process + set FINDSTR= + if not "%%k"=="" set FINDSTR=%%j.%%k + if not "%%l"=="" set FINDSTR=%%j.%%k.%%l %%k.%%l + if not "%%m"=="" set FINDSTR=%%j.%%k.%%l.%%m %%k.%%l.%%m %%l.%%m + if not "%%n"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n %%k.%%l.%%m.%%n %%l.%%m.%%n %%m.%%n + if not "%%o"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o %%k.%%l.%%m.%%n.%%o %%l.%%m.%%n.%%o %%m.%%n.%%o %%n.%%o + if not "%%p"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o.%%p %%k.%%l.%%m.%%n.%%o.%%p %%l.%%m.%%n.%%o.%%p %%m.%%n.%%o.%%p %%n.%%o.%%p %%o.%%p + if not "%%q"=="" set FINDSTR=%%j.%%k.%%l.%%m.%%n.%%o.%%p.%%q %%k.%%l.%%m.%%n.%%o.%%p.%%q %%l.%%m.%%n.%%o.%%p.%%q %%m.%%n.%%o.%%p.%%q %%n.%%o.%%p.%%q %%o.%%p.%%q %%p.%%q + rem If the domain contains 9 parent domains or more, build the search string dynamically + if not "%%r"=="" ( + set PARENTS=%%r + set FINDSTR=%%j %%k %%l %%m %%n %%o %%p %%q !PARENTS:.= ! + set PARENTS= + for %%0 in (!FINDSTR!) do set PARENTS=%%0 !PARENTS! + call :Parents !PARENTS! + ) + rem Cross-check the index for any of the possible parent domains + rem Although using /x for exact match is dramatically slower than partial matches, it's a necessity + rem Phishing domains often stack domain names like XXX.com.YYY.com, and this could create problems for partial matches + findstr /l /x /m "!FINDSTR!" "!CTEMP!" > nul + rem If no parent domains are found in the index, write the domain to file + if !errorlevel!==1 echo local-zone: "%%b" always_nxdomain + ) else echo local-zone: "%%b" always_nxdomain ) - ) + ) else if !COMMENTS!==1 echo !LINE! ) ) > "%~dp0Unbound-%~nx1" + +rem Remove cache files +if exist "!CACHE!" ( + echo Cleaning temporary files... + rmdir /s /q "!CACHE!" +) + echo "%1" converted to "%~dp0Unbound-%~nx1" -pause \ No newline at end of file + +pause + +exit /b + +rem Function for handling domains with 9 parents or more +:Parents +set PARENTS=%2.%1 +set FINDSTR=%2.%1 +shift +shift +:Parents2 +shift +set PARENTS=%0.!PARENTS! +set FINDSTR=!PARENTS! !FINDSTR! +if not "%1"=="" goto Parents2 +exit /b \ No newline at end of file