Test: Lua Hive Partition pager (#6691)

pkg/actions/lua_test.go

@@ -357,6 +357,11 @@ func TestLuaRunTable(t *testing.T) {
 			Input:  "testdata/lua/strings_partition.lua",
 			Output: "testdata/lua/strings_partition.output",
 		},
+		{
+			Name:   "catalogexport_hive_partition_pager",
+			Input:  "testdata/lua/catalogexport_hive_partition_pager.lua",
+			Output: "testdata/lua/catalogexport_hive_partition_pager.output",
+		},
 	}
 
 	for _, testCase := range tests {

pkg/actions/testdata/lua/catalogexport_hive_partition_pager.lua

@@ -0,0 +1,72 @@
+local hive = require("lakefs/catalogexport/hive")
+
+-- helper function to slice table array
+local function slice_dense_array(tbl, first, last, step)
+    local sliced = {}
+
+    for i = first or 1, last or #tbl, step or 1 do
+        sliced[#sliced + 1] = tbl[i]
+    end
+
+    return sliced
+end
+
+-- lakefs mock package
+
+local lakefs = {
+    list_objects = function(repo_id, commit_id, next_offset, prefix, delimiter, page_size)
+        local fs = {
+            [action.repository_id] = {
+                [action.commit_id] = {{
+                    physical_address = "s3://bucket/a1/b1/b",
+                    path = "letters/a=1/b=1/b.csv"
+                }, {
+                    physical_address = "s3://bucket/a2/b2/a",
+                    path = "letters/a=2/b=2/a.csv"
+                }, {
+                    physical_address = "s3://bucket/a2/b2/b",
+                    path = "letters/a=2/b=2/b.csv"
+                }, {
+                    physical_address = "",
+                    path = "letters/a=2/b=3/_SUCCESS"
+                }, {
+                    physical_address = "s3://bucket/a2/b3/a",
+                    path = "letters/a=2/b=3/a.csv"
+                }, {
+                    physical_address = "s3://bucket/a3/b4/a",
+                    path = "letters/a=3/b=4/a.csv"
+                }, {
+                    physical_address = "s3://bucket/a3/b4/b",
+                    path = "letters/a=3/b=4/b.csv"
+                }}
+            }
+        }
+        local all_entries = fs[repo_id][commit_id]
+        if next_offset == "" then
+            next_offset = 1
+        end
+        local end_idx = next_offset + page_size
+        return 200, {
+            results = slice_dense_array(all_entries, next_offset, end_idx),
+            pagination = {
+                has_more = end_idx < #all_entries,
+                next_offset = end_idx + 1
+            }
+        }
+    end
+}
+
+local partitions = {"a", "b"}
+local prefix = "letters/"
+
+for page_size = 1, 10 do
+    local pager = hive.extract_partition_pager(lakefs, action.repository_id, action.commit_id, prefix, partitions,
+        page_size)
+    print("result for page_size " .. tostring(page_size))
+    for part_key, entries in pager do
+        print("# partition: " .. part_key)
+        for _, entry in ipairs(entries) do
+            print("path: " .. entry.path .. " physical: " .. entry.physical_address)
+        end
+    end
+end

pkg/actions/testdata/lua/catalogexport_hive_partition_pager.output

@@ -0,0 +1,110 @@
+result for page_size 1
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 2
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 3
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 4
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 5
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 6
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 7
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 8
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 9
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b
+result for page_size 10
+# partition: letters/a=1/b=1/
+path: letters/a=1/b=1/b.csv physical: s3://bucket/a1/b1/b
+# partition: letters/a=2/b=2/
+path: letters/a=2/b=2/a.csv physical: s3://bucket/a2/b2/a
+path: letters/a=2/b=2/b.csv physical: s3://bucket/a2/b2/b
+# partition: letters/a=2/b=3/
+path: letters/a=2/b=3/a.csv physical: s3://bucket/a2/b3/a
+# partition: letters/a=3/b=4/
+path: letters/a=3/b=4/a.csv physical: s3://bucket/a3/b4/a
+path: letters/a=3/b=4/b.csv physical: s3://bucket/a3/b4/b