diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b75f51b..11ea4e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - name: build - run: make + run: make build_hostfiles - name: upload artifacts uses: actions/upload-artifact@v4 with: @@ -35,7 +35,7 @@ jobs: path: pages/resources/Hosts - name: Make Resource List working-directory: ./pages - run: make + run: make generate - name: Build with Jekyll uses: actions/jekyll-build-pages@v1 with: diff --git a/Makefile b/Makefile index 60bb3ce..3462ab0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ # deployment config +project_owner := kuflierl project_online_home := https://github.com/kuflierl/Blocklists project_rawonline_root := https://kuflierl.github.io/Blocklists/resources @@ -22,11 +23,11 @@ UBLOCK_COMBINELISTS := $(shell find $(COMBINELIST_DIR) -name '*.ublocklist') DOMAIN_COMBINEFINALS := $(patsubst $(COMBINELIST_DIR)/%,$(BUILD_DIR)/%,$(DOMAIN_COMBINELISTS)) define deploader_template = - $(patsubst $(COMBINELIST_DIR)/%,$(BUILD_DIR)/combineobj/%.deploader,$(1)): $(shell sh $(SRC_DIR)/scripts/getdeps-host.sh $(1)) + $(patsubst $(COMBINELIST_DIR)/%,$(BUILD_DIR)/combineobj/%.deploader,$(1)): $(shell sh tools/getdeps-host.sh $(1)) endef .PHONY: all -all: build_hostfiles +all: build_hostfiles build_site $(foreach dcl,$(DOMAIN_COMBINELISTS),$(eval $(call deploader_template,$(dcl)))) @@ -42,18 +43,18 @@ $(BUILD_DIR)/%.host: $(BUILD_DIR)/combineobj/%.host.header $(BUILD_DIR)/combineo $(BUILD_DIR)/combineobj/%.host.header: $(BUILD_DIR)/combineobj/%.host.o mkdir -p $(dir $@) cat $(TEMPLATE_DIR)/host | \ - title="$(@F:%.header=%)" \ + title="$(project_owner)'s $(@F:%.header=%)" \ update_date="$(update_date)" \ update_url="$(project_rawonline_root)/$(*).host" \ project_home="$(project_online_home)" \ uniq_domains=`cat $< | wc -l` \ envsubst > $@ -$(BUILD_DIR)/combineobj/%.host.o: SHELL = /bin/bash $(BUILD_DIR)/combineobj/%.host.o: $(COMBINELIST_DIR)/%.host $(BUILD_DIR)/combineobj/%.host.deploader mkdir -p $(dir $@) touch $@ regex='^domain(black|white)list (text|web)source [\w-]+(\/[\w-]+)*$$'; \ + sed '1!G;h;$$!d' "$<" | \ while IFS="" read -r p || [ -n "$$p" ]; \ do \ echo "$$p" | grep -oPq "$$regex" || continue; \ @@ -63,11 +64,11 @@ $(BUILD_DIR)/combineobj/%.host.o: $(COMBINELIST_DIR)/%.host $(BUILD_DIR)/combine out="$(BUILD_DIR)/$$p2/$$p3.$$p1.o"; \ if [ "$$p1" = 'domainblacklist' ]; \ then sort -mu $@ $$out > $@.tmp; \ - elif [ "$$p1" = 'domainwhitelist' ]; \ - then comm -23 $@ $$out > $@.tmp; \ - fi; \ - mv $@.tmp $@; \ - done <<< `sed '1!G;h;$$!d' "$<"` + elif [ "$$p1" = 'domainwhitelist' ]; \ + then comm -23 $@ $$out > $@.tmp; \ + fi; \ + mv $@.tmp $@; \ + done # sed reverses the lines #todo autogen deploader using eval!! @@ -111,12 +112,29 @@ $(BUILD_DIR)/websource/%.domainwhitelist.o: $(BUILD_DIR)/websource-orig/%.domain mkdir -p $(dir $@) grep -oP '^[ \t]*\d{1,3}(\.\d{1,3}){3}\K([ \t]+[^#!\/ \t\n]*)+' $< | awk 'OFS="\n" {if($$NF > 0) {$$1=$$1;print $$0}}' | sort -u > $@ +.PHONY: generate_site +generate_site: build_hostfiles + mkdir -p pages/resources + cp -r $(BUILD_DIR)/Hosts pages/resources/ + make -C pages generate + +.PHONY: build_site +build_site: generate_site + make -C pages build + .PHONY: build_hostfiles build_hostfiles: $(DOMAIN_COMBINEFINALS) +.PHONY: clean_build +clean_build: + rm -rf $(BUILD_DIR) + +.PHONY: clean_pages +clean_pages: + make -C pages clean + .PHONY: clean -clean: - rm -r $(BUILD_DIR) +clean: clean_build clean_pages .PHONY: dumpvar dumpvar: diff --git a/pages/.ruby-version b/pages/.ruby-version new file mode 100644 index 0000000..eb39e53 --- /dev/null +++ b/pages/.ruby-version @@ -0,0 +1 @@ +3.3 diff --git a/pages/Gemfile.lock b/pages/Gemfile.lock index 5b65a3a..74e50c4 100644 --- a/pages/Gemfile.lock +++ b/pages/Gemfile.lock @@ -55,6 +55,7 @@ GEM pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (5.0.5) + rake (13.2.1) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) @@ -62,8 +63,9 @@ GEM strscan (>= 3.0.9) rouge (4.2.1) safe_yaml (1.0.5) - sass-embedded (1.69.5-x86_64-linux-gnu) + sass-embedded (1.69.5) google-protobuf (~> 3.23) + rake (>= 13.0.0) strscan (3.1.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) diff --git a/pages/Makefile b/pages/Makefile index 756f520..c1640aa 100644 --- a/pages/Makefile +++ b/pages/Makefile @@ -1,8 +1,40 @@ -BUILD_DIR := ./_generated -$(BUILD_DIR)/resources.md: +GENERATE_DIR := ./_generated +BUILD_DIR := ./_site + +.PHONY: all +all: generate build + +.PHONY: generate +generate: $(GENERATE_DIR)/resources.md + +.PHONY: build +build: generate install_deps + bundle exec jekyll build + +.PHONY: install_deps +install_deps: + bundle install + +$(GENERATE_DIR)/resources.md: mkdir -p _generated - sh _make/generate_resources.sh > $(BUILD_DIR)/resources.md + sh _make/generate_resources.sh > $(GENERATE_DIR)/resources.md + +.PHONY: clean_generated +clean_generated: + rm -rf $(GENERATE_DIR) + +.PHONY: clean_site +clean_site: + rm -rf $(BUILD_DIR) .jekyll-cache + +.PHONY: clean_resources +clean_resources: + rm -rf resources .PHONY: clean -clean: - rm -r $(BUILD_DIR) +clean: clean_generated clean_site clean_resources + +# for development +.PHONY: interactive_serve +interactive_serve: + bundle exec jekyll serve -l diff --git a/pages/_make/generate_resources.sh b/pages/_make/generate_resources.sh index ea8e601..42a2183 100644 --- a/pages/_make/generate_resources.sh +++ b/pages/_make/generate_resources.sh @@ -1,4 +1,6 @@ #!/usr/bin/env sh +set -e +[ -d "resources" ] || (echo "find: ‘resources’: No such file or directory" >&2; exit 1) find resources -mindepth 1 -type d | while read folder; do echo "$folder" | awk -F "/" '{while (i++ < NF-1) printf "#"; printf " "; print $NF}' for file in `find "$folder" -maxdepth 1 -type f -printf "%f\n"`; do diff --git a/src/scripts/getdeps-host.sh b/tools/getdeps-host.sh similarity index 100% rename from src/scripts/getdeps-host.sh rename to tools/getdeps-host.sh diff --git a/src/scripts/sorttextlist.py b/tools/sorttextlist.py similarity index 100% rename from src/scripts/sorttextlist.py rename to tools/sorttextlist.py