Add a CI test to generate builds from the JSON api data and check

differences

Author: Musholic

.busted

@@ -9,5 +9,11 @@ return {
         helper = "HeadlessWrapper.lua",
         ROOT = { "../spec" },
        	["exclude-tags"] = "builds",
+    },
+    generate = {
+        directory = "src",
+        lpath = "../runtime/lua/?.lua;../runtime/lua/?/init.lua",
+        helper = "HeadlessWrapper.lua",
+        ROOT = { "../spec/GenerateBuilds.lua" },
     }
 }

.github/workflows/test.yml

@@ -32,3 +32,16 @@ jobs:
       - run: git config --global --add safe.directory $(pwd)
       - name: Check if the generated ModCache is different
         run: git diff --exit-code src/Data/ModCache.lua
+  check_generated_builds:
+    runs-on: ubuntu-latest
+    container: ghcr.io/pathofbuildingcommunity/pathofbuilding-tests:latest
+    steps:
+      - name: Install xmllint and git dependency
+        run: apk add libxml2-utils git
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Generate Builds
+        run: busted --lua=luajit -r generate
+      - run: git config --global --add safe.directory $(pwd)
+      - name: Check if the generated builds are different
+        run: git diff --exit-code spec

docker-compose.yml

@@ -13,3 +13,14 @@ services:
     working_dir: /workdir
     volumes:
       - ./:/workdir:ro
+  busted-generate:
+    image: ghcr.io/pathofbuildingcommunity/pathofbuilding-tests:latest
+    environment:
+      HOME: /tmp
+    container_name: pathofbuilding-generate
+    command: sh -c "apk add libxml2-utils && busted --lua=luajit -r generate"
+    security_opt:
+      - no-new-privileges:true
+    working_dir: /workdir
+    volumes:
+      - ./:/workdir:rw

spec/GenerateBuilds.lua

@@ -0,0 +1,100 @@
+local dkjson = require "dkjson"
+
+local function fetchBuilds(path, buildList)
+	buildList = buildList or {}
+	for file in lfs.dir(path) do
+		if file ~= "." and file ~= ".." then
+			local f = path .. '/' .. file
+			local attr = lfs.attributes(f)
+			assert(type(attr) == "table")
+			if attr.mode == "directory" then
+				fetchBuilds(f, buildList)
+			else
+				if file:match("^.+(%..+)$") == ".json" then
+					local fileHnd, errMsg = io.open(f, "r")
+					if not fileHnd then
+						return nil, errMsg
+					end
+					local fileText = fileHnd:read("*a")
+					fileHnd:close()
+					local fullCharData = dkjson.decode(fileText)
+					buildList[f] = fullCharData.character
+				end
+			end
+		end
+	end
+	return buildList
+end
+
+function buildTable(tableName, values, string, indent)
+	string = string or ""
+	indent = indent or 0
+	local indentStr = string.rep("\t", indent)
+	local nextIndentStr = string.rep("\t", indent + 1)
+	string = string .. indentStr .. tableName .. " = {\n"
+	for key, value in pairsSortByKey(values) do
+		if type(value) == "table" then
+			string = buildTable("[\"" .. key .. "\"]", value, string, indent + 1) .. ",\n"
+		elseif type(value) == "boolean" then
+			string = string .. nextIndentStr .. "[\"" .. key .. "\"] = " .. (value and "true" or "false") .. ",\n"
+		elseif type(value) == "string" then
+			string = string .. nextIndentStr .. "[\"" .. key .. "\"] = \"" .. value .. "\",\n"
+		else
+			string = string .. nextIndentStr .. "[\"" .. key .. "\"] = " .. round(value, 4) .. ",\n"
+		end
+	end
+	string = string .. indentStr .. "}"
+	return string
+end
+
+local function formatXmlFile(filepath)
+	local command = "xmllint --c14n " .. filepath
+
+	-- Open the command process for reading ('r')
+	local handle = io.popen(command, 'r')
+	if not handle then
+		return nil, "Failed to run xmllint. Is it installed and in your PATH?"
+	end
+
+	-- Read the entire output from the command
+	local result = handle:read("*a")
+	handle:close()
+
+	local fileHnd, errMsg = io.open(filepath:gsub("-unformatted", ""), "w")
+	fileHnd:write(result)
+	fileHnd:close()
+end
+
+expose("generate all builds", function()
+	local buildList = fetchBuilds("../spec/TestBuilds")
+	for filename, testBuild in pairs(buildList) do
+		local buildName = filename:gsub("^.*/([^/]+/[^/]+)%.[^.]+$", "%1")
+
+		it("on build: " .. buildName, function()
+			-- Load the build and calculate stats
+			print("Loading build: " .. buildName)
+			loadBuildFromJSON(testBuild, testBuild)
+
+			print("Saving main output")
+			-- Save the calculated output to a Lua file in the generated sub-folder
+			-- Get rid of circular reference issues first
+			build.calcsTab.mainOutput.ReqDexItem = nil
+			build.calcsTab.mainOutput.ReqIntItem = nil
+			build.calcsTab.mainOutput.ReqStrItem = nil
+			local outputFile = filename:gsub("^(.+)%..+$", "%1.lua")
+			outputFile = outputFile:gsub("^(.+)/([^/]+)$", "%1/generated/%2")
+			local fileHnd, errMsg = io.open(outputFile, "w+")
+			fileHnd:write(buildTable("output", build.calcsTab.mainOutput))
+			fileHnd:close()
+
+			-- Save the XML DB file in the generated sub-folder
+			print("Saving xml db file")
+			build.dbFileName = outputFile:gsub("^(.+)%..+$", "%1-unformatted.xml")
+			build:SaveDBFile()
+			-- Format/order the XML file to easily see differences with previous generations
+			formatXmlFile(build.dbFileName)
+			-- Remove the unformatted XML file
+			os.remove(build.dbFileName)
+		end)
+	end
+end)