From e410666e5855325732ce9027f1fab3149bc16c1f Mon Sep 17 00:00:00 2001 From: teknomunk Date: Mon, 29 Apr 2024 11:15:29 +0000 Subject: [PATCH] Initial map indexer implementation --- mods/CORE/vl_map_index/api.lua | 12 +++ mods/CORE/vl_map_index/async_api.lua | 35 +++++++ mods/CORE/vl_map_index/init.lua | 133 ++++++++++++++++++++++++ mods/CORE/vl_map_index/mod.conf | 5 + mods/CORE/vl_map_index/node_counter.lua | 71 +++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 mods/CORE/vl_map_index/api.lua create mode 100644 mods/CORE/vl_map_index/async_api.lua create mode 100644 mods/CORE/vl_map_index/init.lua create mode 100644 mods/CORE/vl_map_index/mod.conf create mode 100644 mods/CORE/vl_map_index/node_counter.lua diff --git a/mods/CORE/vl_map_index/api.lua b/mods/CORE/vl_map_index/api.lua new file mode 100644 index 000000000..ba164bf9d --- /dev/null +++ b/mods/CORE/vl_map_index/api.lua @@ -0,0 +1,12 @@ +dofile(modpath.."/async_api.lua") +minetest.register_async_dofile(modpath.."/async_api.lua") + +local mod = vl_map_index +mod.main_thread = true + +function mod.get_indexes_for_area(pos1, pos2, index_name) + return {} +end +function mod.get_indexes_near(pos1, radius, index_name) + return {} +end diff --git a/mods/CORE/vl_map_index/async_api.lua b/mods/CORE/vl_map_index/async_api.lua new file mode 100644 index 000000000..f478f47e9 --- /dev/null +++ b/mods/CORE/vl_map_index/async_api.lua @@ -0,0 +1,35 @@ +vl_map_index = { + indexers = {}, + indexer_names = {}, +} +local mod = vl_map_index + +function mod.indexer_dofile(filename) + if minetest.register_async_dofile then + minetest.register_async_dofile(filename) + end + dofile(filename) +end + +-- Registration +function mod.register_indexer(name, callback) + if vl_map_index.indexers[name] then + minetest.log("warning", "[vl_map_index] Indexer "..tostring(name).." already defined. vl_map_index.register_indexer called from "..debug.traceback()) + return + end + + local indexer_names = mod.indexer_names + indexer_names[#indexer_names + 1] = name + mod.indexers[name] = callback + + print("[vl_map_index] Registered indexer "..name.."\n indexer_names: "..dump(indexer_names)) +end + +-- Index helpers +function mod.pack_index_data(data) + return minetest.compress(minetest.serialize(data),"deflate") +end +function mod.unpack_index_data(str) + return minetest.deserialize(minetest.decompress(str,"deflate")) +end + diff --git a/mods/CORE/vl_map_index/init.lua b/mods/CORE/vl_map_index/init.lua new file mode 100644 index 000000000..006545feb --- /dev/null +++ b/mods/CORE/vl_map_index/init.lua @@ -0,0 +1,133 @@ +local modname = minetest.get_current_modname() +local modpath = minetest.get_modpath(modname) + +-- Make sure the API is available everywhere +dofile(modpath.."/api.lua") +local mod = vl_map_index +mod.indexer_dofile(modpath.."/node_counter.lua") + +-- Imports +local call_async = minetest.handle_async or function(a,b,...) b(a(...)) end + +-- Constants +local DIRTY_BLOCK_HOLDOFF = 2 +local MINETEST_BLOCK_SIZE = 16 + +-- Load and save data +local storage = minetest.get_mod_storage() +local dirty_blocks = minetest.deserialize(storage:get_string("dirty_blocks")) or {} +local processing_block = minetest.deserialize(storage:get_string("processing_block")) or false +minetest.register_on_shutdown(function() + storage:set_string("dirty_blocks", minetest.serialize(dirty_blocks)) + storage:set_string("processing_block",minetest.serialize(processing_block)) +end) + +-- Map block processor +local remaining_indexers = nil +local index_in_process = false +local block_data = nil +local function process_dirty_block() + if not processing_block then return false end + if index_in_process then return false end + + -- If we are starting + if not remaining_indexers then + print("Starting to index "..vector.to_string(minetest.get_position_from_hash(processing_block))) + remaining_indexers = table.copy(mod.indexer_names) + + -- Use a 3x3x3 for indexing, to allow access to neighbors + -- Do this only once per indexing operation + local vm = minetest.get_voxel_manip() + local map_block_anchor = vector.multiply(minetest.get_position_from_hash(processing_block), MINETEST_BLOCK_SIZE) + local min = vector.offset(map_block_anchor, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE) + local max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1) + local emin,emax = vm:read_from_map(min, max) + local data = vm:get_data() + block_data = { + map_block = { + hash = processing_block, + min = map_block_anchor, + max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1) + }, + vm = vm, + emin = emin, emax = emax, + vm_data = data, + } + end + + -- Get the next index to process + local idx = #remaining_indexers + if idx == 0 then + -- Finished + print("Finished indexing "..vector.to_string(minetest.get_position_from_hash(processing_block))) + processing_block = nil + remaining_indexers = nil + block_data = nil + return false + end + local indexer = remaining_indexers[idx] + remaining_indexers[idx] = nil + + -- Run the indexer + local function process(data) + return vl_map_index.pack_index(vl_map_index.indexers[data.indexer](data)), data.indexer, data.map_block.hash + end + local function finish(result, indexer, map_block_hash) + storage:set_string(indexer..tostring(vector.to_string(minetest.get_position_from_hash(map_block_hash))),tostring(result)) + index_in_process = false + end + + -- Run the indexer + index_in_process = true + block_data.indexer = indexer + print("Running index "..indexer) + call_async(process, finish, block_data) + + return true +end + +-- Flag a map block as changed +function mod.flag_map_block_dirty(pos) + local map_block = vector.new( + math.floor(pos.x / MINETEST_BLOCK_SIZE), + math.floor(pos.y / MINETEST_BLOCK_SIZE), + math.floor(pos.z / MINETEST_BLOCK_SIZE) + ) + local hash = minetest.hash_node_position(map_block) + dirty_blocks[hash] = minetest.get_us_time() +end +function mod.index_around_entity(pos, distance) + +end + +-- Flag any map block that has nodes placed or dug for an update +minetest.register_on_placenode(function(pos, newnode, placer, oldnode, itemstack, pointed_thing) + mod.flag_map_block_dirty(pos) +end) +minetest.register_on_dignode(function(pos, oldnode, placer) + mod.flag_map_block_dirty(pos) +end) + +-- Periodic task to process dirty map blocks +local function run_indexer() + -- Do nothing if a map block is being processed asynchronously + if process_dirty_block() then return end + + -- Process the first dirty blocks that hasn't been touched recently + local holdoff_time_check = minetest.get_us_time() - DIRTY_BLOCK_HOLDOFF * 1e6 + for block_hash,holdoff_time in pairs(dirty_blocks) do + if holdoff_time < holdoff_time_check then + processing_block = block_hash + dirty_blocks[block_hash] = nil + return + end + end +end +if minetest.get_modpath("vl_scheduler") then + -- Run the indexer + vl_scheduler.register_periodic_task("vl_map_index:indexer", 0.25, 4, run_indexer) +else + -- Globalstep to handle reindexing map blocks + minetest.register_globalstep(run_indexer) +end + diff --git a/mods/CORE/vl_map_index/mod.conf b/mods/CORE/vl_map_index/mod.conf new file mode 100644 index 000000000..27da70365 --- /dev/null +++ b/mods/CORE/vl_map_index/mod.conf @@ -0,0 +1,5 @@ +name = vl_map_index +author = teknomunk +description = Map data index +depends = mcl_util +optional_depends = vl_scheduler diff --git a/mods/CORE/vl_map_index/node_counter.lua b/mods/CORE/vl_map_index/node_counter.lua new file mode 100644 index 000000000..b535280e4 --- /dev/null +++ b/mods/CORE/vl_map_index/node_counter.lua @@ -0,0 +1,71 @@ +-- Imports +local vl_map_index = vl_map_index + +vl_map_index.register_indexer("node_counter",function(data) + local min = data.map_block.min + local max = data.map_block.max + local area = VoxelArea:new{ MinEdge = data.emin, MaxEdge = data.emax } + local vm_data = data.vm_data + + local counts = {} + + -- Count all nodes + for z = min.z,max.z do + for y = min.y,max.y do + for x = min.x,max.x do + local idx = area:index(x,y,z) + local cid = vm_data[idx] + counts[cid] = (counts[cid] or 0) + 1 + end + end + end + + -- Convert counts to use node names instead of content ids + local node_counts = {} + for cid,count in pairs(counts) do + local node_name = minetest.get_name_from_content_id(cid) + node_counts[node_name] = count + end + + -- This is our index + return { + node_counts = counts, + } +end) + +-- Public-facing API +if vl_map_index.main_thread then + function vl_map_index.find_nodes_in_area(pos1, pos2, nodenames, grouped) + local indexes = vl_map_index.get_indexes_for_area(pos1, pos2, "node_counter") + if #indexes > 0 then + for i=1,#indexes do + -- TODO: find nodes in area + end + end + + -- Otherwize, fallback onto minetest builtin function + return minetest.find_nodes_in_area(pos1, pos2, nodenames, grouped) + end + function vl_map_index.find_nodes_near(pos, radius, nodenames, search_center) + local indexes = vl_map_index.get_indexes_near(pos, radius, "node_counter") + if #indexes > 0 then + for i=1,#indexes do + -- TODO: find nodes in area + end + end + + -- Otherwize, fallback onto minetest builtin function + return minetest.find_nodes_near(pos, radius, nodenames, search_center) + end + function vl_map_index.find_nodes_in_area_under_air(pos1, pos2, nodenames) + local indexes = vl_map_index.get_indexes_for_area(pos1, pos2, "node_counter") + if #indexes > 0 then + for i=1,#indexes do + -- TODO: find nodes under air in area + end + end + + -- Otherwize, fallback onto minetest builtin function + return minetest.find_nodes_in_area_under_air(pos1, pos2, nodenames) + end +end