Initial map indexer implementation

mods/CORE/vl_map_index/api.lua

@@ -0,0 +1,12 @@
+dofile(modpath.."/async_api.lua")
+minetest.register_async_dofile(modpath.."/async_api.lua")
+
+local mod = vl_map_index
+mod.main_thread = true
+
+function mod.get_indexes_for_area(pos1, pos2, index_name)
+	return {}
+end
+function mod.get_indexes_near(pos1, radius, index_name)
+	return {}
+end

mods/CORE/vl_map_index/async_api.lua

@@ -0,0 +1,35 @@
+vl_map_index = {
+	indexers = {},
+	indexer_names = {},
+}
+local mod = vl_map_index
+
+function mod.indexer_dofile(filename)
+	if minetest.register_async_dofile then
+		minetest.register_async_dofile(filename)
+	end
+	dofile(filename)
+end
+
+-- Registration
+function mod.register_indexer(name, callback)
+	if vl_map_index.indexers[name] then
+		minetest.log("warning", "[vl_map_index] Indexer "..tostring(name).." already defined. vl_map_index.register_indexer called from "..debug.traceback())
+		return
+	end
+
+	local indexer_names = mod.indexer_names
+	indexer_names[#indexer_names + 1] = name
+	mod.indexers[name] = callback
+
+	print("[vl_map_index] Registered indexer "..name.."\n indexer_names: "..dump(indexer_names))
+end
+
+-- Index helpers
+function mod.pack_index_data(data)
+	return minetest.compress(minetest.serialize(data),"deflate")
+end
+function mod.unpack_index_data(str)
+	return minetest.deserialize(minetest.decompress(str,"deflate"))
+end
+

mods/CORE/vl_map_index/init.lua

@@ -0,0 +1,133 @@
+local modname = minetest.get_current_modname()
+local modpath = minetest.get_modpath(modname)
+
+-- Make sure the API is available everywhere
+dofile(modpath.."/api.lua")
+local mod = vl_map_index
+mod.indexer_dofile(modpath.."/node_counter.lua")
+
+-- Imports
+local call_async = minetest.handle_async or function(a,b,...) b(a(...)) end
+
+-- Constants
+local DIRTY_BLOCK_HOLDOFF = 2
+local MINETEST_BLOCK_SIZE = 16
+
+-- Load and save data
+local storage = minetest.get_mod_storage()
+local dirty_blocks = minetest.deserialize(storage:get_string("dirty_blocks")) or {}
+local processing_block = minetest.deserialize(storage:get_string("processing_block")) or false
+minetest.register_on_shutdown(function()
+	storage:set_string("dirty_blocks", minetest.serialize(dirty_blocks))
+	storage:set_string("processing_block",minetest.serialize(processing_block))
+end)
+
+-- Map block processor
+local remaining_indexers = nil
+local index_in_process = false
+local block_data = nil
+local function process_dirty_block()
+	if not processing_block then return false end
+	if index_in_process then return false end
+
+	-- If we are starting
+	if not remaining_indexers then
+		print("Starting to index "..vector.to_string(minetest.get_position_from_hash(processing_block)))
+		remaining_indexers = table.copy(mod.indexer_names)
+
+		-- Use a 3x3x3 for indexing, to allow access to neighbors
+		-- Do this only once per indexing operation
+		local vm = minetest.get_voxel_manip()
+		local map_block_anchor = vector.multiply(minetest.get_position_from_hash(processing_block), MINETEST_BLOCK_SIZE)
+		local min = vector.offset(map_block_anchor, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE)
+		local max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1)
+		local emin,emax = vm:read_from_map(min, max)
+		local data = vm:get_data()
+		block_data = {
+			map_block = {
+				hash = processing_block,
+				min = map_block_anchor,
+				max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1)
+			},
+			vm = vm,
+			emin = emin, emax = emax,
+			vm_data = data,
+		}
+	end
+
+	-- Get the next index to process
+	local idx = #remaining_indexers
+	if idx == 0 then
+		-- Finished
+		print("Finished indexing "..vector.to_string(minetest.get_position_from_hash(processing_block)))
+		processing_block = nil
+		remaining_indexers = nil
+		block_data = nil
+		return false
+	end
+	local indexer = remaining_indexers[idx]
+	remaining_indexers[idx] = nil
+
+	-- Run the indexer
+	local function process(data)
+		return vl_map_index.pack_index(vl_map_index.indexers[data.indexer](data)), data.indexer, data.map_block.hash
+	end
+	local function finish(result, indexer, map_block_hash)
+		storage:set_string(indexer..tostring(vector.to_string(minetest.get_position_from_hash(map_block_hash))),tostring(result))
+		index_in_process = false
+	end
+
+	-- Run the indexer
+	index_in_process = true
+	block_data.indexer = indexer
+	print("Running index "..indexer)
+	call_async(process, finish, block_data)
+
+	return true
+end
+
+-- Flag a map block as changed
+function mod.flag_map_block_dirty(pos)
+	local map_block = vector.new(
+		math.floor(pos.x / MINETEST_BLOCK_SIZE),
+		math.floor(pos.y / MINETEST_BLOCK_SIZE),
+		math.floor(pos.z / MINETEST_BLOCK_SIZE)
+	)
+	local hash = minetest.hash_node_position(map_block)
+	dirty_blocks[hash] = minetest.get_us_time()
+end
+function mod.index_around_entity(pos, distance)
+
+end
+
+-- Flag any map block that has nodes placed or dug for an update
+minetest.register_on_placenode(function(pos, newnode, placer, oldnode, itemstack, pointed_thing)
+	mod.flag_map_block_dirty(pos)
+end)
+minetest.register_on_dignode(function(pos, oldnode, placer)
+	mod.flag_map_block_dirty(pos)
+end)
+
+-- Periodic task to process dirty map blocks
+local function run_indexer()
+	-- Do nothing if a map block is being processed asynchronously
+	if process_dirty_block() then return end
+
+	-- Process the first dirty blocks that hasn't been touched recently
+	local holdoff_time_check = minetest.get_us_time() - DIRTY_BLOCK_HOLDOFF * 1e6
+	for block_hash,holdoff_time in pairs(dirty_blocks) do
+		if holdoff_time < holdoff_time_check then
+			processing_block = block_hash
+			dirty_blocks[block_hash] = nil
+			return
+		end
+	end
+end
+if minetest.get_modpath("vl_scheduler") then
+	-- Run the indexer
+	vl_scheduler.register_periodic_task("vl_map_index:indexer", 0.25, 4, run_indexer)
+else
+	-- Globalstep to handle reindexing map blocks
+	minetest.register_globalstep(run_indexer)
+end
+

mods/CORE/vl_map_index/mod.conf

@@ -0,0 +1,5 @@
+name = vl_map_index
+author = teknomunk
+description = Map data index
+depends = mcl_util
+optional_depends = vl_scheduler

mods/CORE/vl_map_index/node_counter.lua

@@ -0,0 +1,71 @@
+-- Imports
+local vl_map_index = vl_map_index
+
+vl_map_index.register_indexer("node_counter",function(data)
+	local min = data.map_block.min
+	local max = data.map_block.max
+	local area = VoxelArea:new{ MinEdge = data.emin, MaxEdge = data.emax }
+	local vm_data = data.vm_data
+
+	local counts = {}
+
+	-- Count all nodes
+	for z = min.z,max.z do
+		for y = min.y,max.y do
+			for x = min.x,max.x do
+				local idx = area:index(x,y,z)
+				local cid = vm_data[idx]
+				counts[cid] = (counts[cid] or 0) + 1
+			end
+		end
+	end
+
+	-- Convert counts to use node names instead of content ids
+	local node_counts = {}
+	for cid,count in pairs(counts) do
+		local node_name = minetest.get_name_from_content_id(cid)
+		node_counts[node_name] = count
+	end
+
+	-- This is our index
+	return {
+		node_counts = counts,
+	}
+end)
+
+-- Public-facing API
+if vl_map_index.main_thread then
+	function vl_map_index.find_nodes_in_area(pos1, pos2, nodenames, grouped)
+		local indexes = vl_map_index.get_indexes_for_area(pos1, pos2, "node_counter")
+		if #indexes > 0 then
+			for i=1,#indexes do
+				-- TODO: find nodes in area
+			end
+		end
+
+		-- Otherwize, fallback onto minetest builtin function
+		return minetest.find_nodes_in_area(pos1, pos2, nodenames, grouped)
+	end
+	function vl_map_index.find_nodes_near(pos, radius, nodenames, search_center)
+		local indexes = vl_map_index.get_indexes_near(pos, radius, "node_counter")
+		if #indexes > 0 then
+			for i=1,#indexes do
+				-- TODO: find nodes in area
+			end
+		end
+
+		-- Otherwize, fallback onto minetest builtin function
+		return minetest.find_nodes_near(pos, radius, nodenames, search_center)
+	end
+	function vl_map_index.find_nodes_in_area_under_air(pos1, pos2, nodenames)
+		local indexes = vl_map_index.get_indexes_for_area(pos1, pos2, "node_counter")
+		if #indexes > 0 then
+			for i=1,#indexes do
+				-- TODO: find nodes under air in area
+			end
+		end
+
+		-- Otherwize, fallback onto minetest builtin function
+		return minetest.find_nodes_in_area_under_air(pos1, pos2, nodenames)
+	end
+end