Quiet things up

mods/CORE/vl_map_index/api.lua

@@ -0,0 +1,82 @@
+local modname = minetest.get_current_modname()
+local modpath = minetest.get_modpath(modname)
+dofile(modpath.."/async_api.lua")
+minetest.register_async_dofile(modpath.."/async_api.lua")
+local storage = minetest.get_mod_storage()
+
+local mod = vl_map_index
+mod.main_thread = true
+
+local cache = {}
+function mod.get_index(key)
+	local cached = cache[key]
+	if cached then return cached end
+
+	local value = storage:get_string(key)
+	if value ~= "" then
+		local data = mod.unpack_index(value)
+		cache[key] = data
+		return data
+	end
+
+	return nil
+end
+function mod.clear_cache(key)
+	cache[key] = nil
+end
+
+
+local index_cache = {}
+local index_cache_count = 0
+function mod.cache_map_index(map_block_hash, indexer, result)
+	assert(map_block_hash)
+
+	local block_cache = index_cache[map_block_hash]
+	if not block_cache then
+		if index_cache_count > 64 then
+			-- evict something
+			local to_evict = math.random(0,index_cache_count)
+			for k,_ in pairs(index_cache) do
+				if to_evict == 0 then
+					index_cache[k] = nil
+					index_cache_count = index_cache_count - 1
+					break
+				end
+				to_evict = to_evict - 1
+			end
+		end
+
+		block_cache = {}
+		index_cache[map_block_hash] = block_cache
+		index_cache_count = index_cache_count + 1
+	end
+
+	block_cache[indexer] = result
+end
+function mod.get_index_for_map_block(map_block, indexer)
+	local map_block_hash = minetest.hash_node_position(map_block)
+
+	-- Use the cached version if available
+	local cached = index_cache[map_block_hash]
+	if cached then return cached[indexer] end
+
+	-- Try to load from mod storage
+	local key = indexer..tostring(vector.to_string(map_block))
+	local value = storage:get_string(key)
+	if value ~= "" then
+		local index_data = mod.unpack_index(value)
+		mod.cache_map_index(map_block_hash, indexer, index_data)
+		return index_data
+	end
+
+	-- We don't have any data, queue it up and return failure
+	mod.flag_map_block_hash_dirty(map_block_hash, true)
+	return nil
+end
+
+function mod.get_indexes_for_area(pos1, pos2, index_name)
+	return nil
+end
+function mod.get_indexes_near(pos1, radius, index_name)
+	return nil
+end

mods/CORE/vl_map_index/async_api.lua

@@ -0,0 +1,53 @@
+vl_map_index = {
+	indexers = {},
+	indexer_ids = {},
+}
+local mod = vl_map_index
+
+-- Constants
+local MINETEST_BLOCK_SIZE = 16
+
+function mod.indexer_dofile(filename)
+	if minetest.register_async_dofile then
+		minetest.register_async_dofile(filename)
+	end
+	dofile(filename)
+end
+
+-- Registration
+function mod.register_indexer(name, def)
+	if vl_map_index.indexers[name] then
+		minetest.log("warning", "[vl_map_index] Indexer "..tostring(name).." already defined. vl_map_index.register_indexer called from "..debug.traceback())
+		return
+	end
+
+	local indexers = mod.indexers
+	local id = #indexers + 1
+	def.id = id
+	def.name = name
+	indexers[id] = def
+	mod.indexer_ids[name] = id
+
+	print("[vl_map_index] Registered indexer "..name.."\n indexer_names: "..dump(indexers))
+end
+
+function mod.map_block(pos)
+	return vector.new(
+		math.floor(pos.x / MINETEST_BLOCK_SIZE),
+		math.floor(pos.y / MINETEST_BLOCK_SIZE),
+		math.floor(pos.z / MINETEST_BLOCK_SIZE)
+	)
+end
+
+-- Index helpers
+function mod.pack_index(data)
+	return minetest.encode_base64(minetest.compress(minetest.serialize(data),"deflate"))
+end
+function mod.unpack_index(str)
+	if not str or str == "" then return nil end
+	local decoded = minetest.decode_base64(str)
+	if not decoded then return nil end
+
+	return minetest.deserialize(minetest.decompress(decoded,"deflate"))
+end
+

mods/CORE/vl_map_index/init.lua

@@ -0,0 +1,239 @@
+local modname = minetest.get_current_modname()
+local modpath = minetest.get_modpath(modname)
+
+-- Make sure the API is available everywhere
+dofile(modpath.."/api.lua")
+if not minetest.get_modpath("vl_scheduler") then
+	dofile(modpath.."/scheduler.lua")
+end
+local mod = vl_map_index
+mod.indexer_dofile(modpath.."/node_counter.lua")
+
+
+-- Imports
+local call_async = minetest.handle_async or function(a,b,...) b(a(...)) end
+local scheduler = mod.scheduler
+
+-- Constants
+local DIRTY_BLOCK_HOLDOFF = 2
+local MINETEST_BLOCK_SIZE = 16
+
+-- Load and save data
+local storage = minetest.get_mod_storage()
+local dirty_blocks = minetest.deserialize(storage:get_string("dirty_blocks")) or {}
+local processing_block = minetest.deserialize(storage:get_string("processing_block")) or false
+local post_process_blocks = minetest.deserialize(storage:get_string("post_process_blocks")) or {}
+minetest.register_on_shutdown(function()
+	storage:set_string("dirty_blocks", minetest.serialize(dirty_blocks))
+	storage:set_string("processing_block",minetest.serialize(processing_block))
+	storage:set_string("pos_process_blocks",minetest.serialize(post_process_blocks))
+end)
+
+-- Map block processor
+local decompress_queue = {}
+local remaining_indexers = nil
+local index_in_process = false
+local block_data = nil
+local function process_dirty_block()
+	if not processing_block then return false end
+	if index_in_process then return false end
+
+	-- If we are starting
+	if not remaining_indexers then
+		--print("Starting to index "..vector.to_string(minetest.get_position_from_hash(processing_block)))
+		remaining_indexers = #mod.indexers
+
+		-- Use a 3x3x3 for indexing, to allow access to neighbors
+		-- Do this only once per indexing operation
+		local vm = minetest.get_voxel_manip()
+		local map_block = minetest.get_position_from_hash(processing_block)
+		local map_block_anchor = vector.multiply(map_block, MINETEST_BLOCK_SIZE)
+		local min = vector.offset(map_block_anchor, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE, -MINETEST_BLOCK_SIZE)
+		local max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1, MINETEST_BLOCK_SIZE * 2 - 1)
+		local emin,emax = vm:read_from_map(min, max)
+		block_data = {
+			map_block = {
+				hash = processing_block,
+				pos = map_block,
+				min = map_block_anchor,
+				max = vector.offset(map_block_anchor, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1, MINETEST_BLOCK_SIZE - 1)
+			},
+			vm = vm,
+			emin = emin, emax = emax,
+		}
+	end
+
+	-- Get the next index to process
+	if remaining_indexers == 0 then
+		-- Finished
+		--print("Finished indexing "..vector.to_string(minetest.get_position_from_hash(processing_block)))
+		processing_block = nil
+		remaining_indexers = nil
+		block_data = nil
+		return false
+	end
+	local indexer = remaining_indexers
+	remaining_indexers = remaining_indexers - 1
+
+	-- Run the indexer
+	local function process(data)
+		local indexer = vl_map_index.indexers[data.indexer]
+
+		return vl_map_index.pack_index(indexer.index(data)), data.indexer, data.map_block.hash
+	end
+	local function finish(result, indexer, map_block_hash)
+		-- Flag the map block for post processing
+		local post_processor = mod.indexers[indexer].post_process
+		if post_processor then
+			result = post_processor(map_block_hash, result)
+		end
+		local key = block_data.index_string
+		storage:set_string(key, tostring(result))
+		mod.clear_cache(key)
+
+		index_in_process = false
+		decompress_queue[#decompress_queue + 1] = {map_block_hash, indexer, result}
+		process_dirty_block()
+	end
+
+	-- Run the indexer
+	index_in_process = true
+	block_data.indexer = indexer
+
+	-- Get the old indexed data
+	local index_string = mod.indexers[indexer].name..vector.to_string(block_data.map_block.pos)
+	block_data.old_index = storage:get_string(index_string)
+	block_data.index_string = index_string
+
+	vl_map_index.indexers[indexer].pre_process(block_data)
+	--print("Running index "..mod.indexers[indexer].name)
+	call_async(process, finish, block_data)
+
+	return true
+end
+
+-- Flag a map block as changed
+function mod.flag_position_dirty(pos)
+	local map_block = vector.new(
+		math.floor(pos.x / MINETEST_BLOCK_SIZE),
+		math.floor(pos.y / MINETEST_BLOCK_SIZE),
+		math.floor(pos.z / MINETEST_BLOCK_SIZE)
+	)
+	local hash = minetest.hash_node_position(map_block)
+	if not dirty_blocks[hash] then
+		--print("Adding "..vector.to_string(map_block).." to dirty set")
+	end
+	dirty_blocks[hash] = minetest.get_us_time()
+end
+function mod.flag_map_block_hash_dirty(map_block_hash, force)
+	if force then
+		dirty_blocks[map_block_hash] = 0
+	else
+		dirty_blocks[map_block_hash] = minetest.get_us_time()
+	end
+end
+function mod.flag_position_hash_dirty(pos_hash)
+	local pos = minetest.get_position_from_hash(pos_hash)
+	mod.flag_position_dirty(pos)
+end
+-- Returns true if all indexes have been generated for map_block
+function mod.has_index(map_block)
+	for i = 1,#mod.indexers do
+		local idx = mod.indexers[i].name..vector.to_string(map_block)
+		if not storage:contains(idx) then
+			return false
+		end
+	end
+
+	return true
+end
+function mod.ensure_indexed(map_block)
+	if not mod.has_index(map_block) then
+		dirty_blocks[minetest.hash_node_position(map_block)] = 0
+	end
+end
+function mod.index_around_entity(pos, distance)
+	local center = mod.map_block(pos)
+	local blocks = math.ceil(distance / MINETEST_BLOCK_SIZE)
+
+	for z = -blocks,blocks do
+		for y = -2,1 do
+			for x = -blocks,blocks do
+				mod.ensure_indexed(vector.add(center, vector.new(x,y,z)))
+			end
+		end
+	end
+end
+
+-- Flag any map block that has nodes placed or dug for an update
+minetest.register_on_placenode(function(pos, newnode, placer, oldnode, itemstack, pointed_thing)
+	mod.flag_position_dirty(pos)
+end)
+minetest.register_on_dignode(function(pos, oldnode, placer)
+	mod.flag_position_dirty(pos)
+end)
+minetest.register_on_generated(function(minp, maxp, blockseed)
+	mod.flag_position_dirty(minp)
+end)
+
+--[[
+minetest.register_on_mapblocks_changed(function(modified_blocks, modified_block_count)
+	for hash,_ in pairs(modified_blocks) do
+		mod.flag_position_hash_dirty(hash)
+	end
+	print(dump(dirty_blocks))
+end)
+--]]
+
+
+-- Periodic task to process dirty map blocks
+local last_player_blocks = {}
+local function run_indexer()
+	-- Handle player positions
+	local players = minetest.get_connected_players()
+	for i=1,#players do
+		local player = players[i]
+		local player_pos = player:get_pos()
+		local map_block = minetest.hash_node_position(mod.map_block(player_pos))
+		local player_name = player:get_player_name()
+
+		local last_player_block = last_player_blocks[player_name]
+		if not last_player_block or last_player_block ~= map_block then
+			--print("Indexing around "..player_name)
+			mod.index_around_entity(player_pos, 128)
+		end
+		last_player_blocks[player_name] = map_block
+	end
+
+	-- Do nothing if a map block is being processed asynchronously
+	if process_dirty_block() then return end
+
+	-- Process the first dirty blocks that hasn't been touched recently
+	local holdoff_time_check = minetest.get_us_time() - DIRTY_BLOCK_HOLDOFF * 1e6
+	for block_hash,holdoff_time in pairs(dirty_blocks) do
+		if holdoff_time < holdoff_time_check then
+			processing_block = block_hash
+			dirty_blocks[block_hash] = nil
+			process_dirty_block()
+			return
+		end
+	end
+
+	-- There are no blocks to index, try to decompress a previously indexed block
+	local idx = #decompress_queue
+	local item = decompress_queue[idx]
+	decompress_queue[idx] = nil
+	if item then
+		local map_block_hash = item[1]
+		local indexer = item[2]
+		mod.cache_map_index(map_block_hash, indexer, mod.unpack_index(item[3]))
+		if post_process_blocks[item[1]] then
+			mod.indexers[indexer].post_process(map_block_hash)
+			post_process_blocks[item[1]] = nil
+		end
+	end
+end
+
+-- Run the indexer
+scheduler.register_periodic_task("vl_map_index:indexer", 0.25, 4, run_indexer)
+

mods/CORE/vl_map_index/mod.conf

@@ -0,0 +1,5 @@
+name = vl_map_index
+author = teknomunk
+description = Map data index
+depends = mcl_util
+optional_depends = vl_scheduler

mods/CORE/vl_map_index/node_counter.lua

@@ -0,0 +1,509 @@
+-- Imports
+local mod = vl_map_index
+local storage
+if mod.main_thread then
+	storage = minetest.get_mod_storage()
+end
+
+-- Constants
+local MIN_SEARCH_SIZE = 1000
+
+local function block_pos_hash_vector(min, pos)
+	return (pos.x - min.x) * 10000 +
+	       (pos.y - min.y) * 100   +
+	       (pos.z - min.z)
+end
+local function block_pos_hash(min, x, y, z)
+	return (x - min.x) * 10000 +
+	       (y - min.y) * 100   +
+	       (z - min.z)
+end
+local function get_block_pos_from_hash(min, hash)
+	local dz = hash % 100; hash = (hash - dz) / 100
+	local dy = hash % 100; hash = (hash - dy) / 100
+	local dx = hash % 100
+	return vector.offset(min, dx, dy, dz)
+end
+
+local l2_tasks = {}
+local l3_tasks = {}
+local l4_task = nil
+
+local function bitmap_index_set(index, offset)
+	local bit_pos = bit.lshift(1, offset % 32)
+	local part_idx = math.floor(offset / 32)
+
+	local part = index[part_idx] or 0
+	part = bit.bor(part, bit_pos)
+
+	index[part_idx] = part
+end
+local function bitmap_index_clear(index, offset)
+	local bit_pos = bit.lshift(1, offset % 32)
+	local bit_pos_neg = 0xFFFFFFFF - bit_pos
+
+	local part_idx = math.floor(offset / 32)
+	local part = index[part_idx] or 0
+
+	part = bit.band(part, bit_pos_neg)
+	if part == 0 then
+		index[part_idx] = nil
+	else
+		index[part_idx] = part
+	end
+end
+
+local function make_lower_to_upper_cid_map(lower_data, upper_data)
+	local upper_cid_map = upper_data.cid_map or {}
+	upper_data.cid_map = upper_cid_map
+
+	-- Build CID conversion table
+	local upper_cid_from_node_name = {}
+	local max_upper_cid = 0
+	for cid,name in pairs(upper_cid_map or {}) do
+		upper_cid_from_node_name[name] = cid
+		if cid > max_upper_cid then
+			max_upper_cid = cid
+		end
+	end
+
+	local lower_to_upper_cid_map = {}
+	for cid,name in pairs(lower_data.cid_map or {}) do
+		local upper_cid = upper_cid_from_node_name[name]
+		if not upper_cid then
+			max_upper_cid = max_upper_cid + 1
+			upper_cid = max_upper_cid
+
+			upper_cid_map[upper_cid] = name
+		end
+		lower_to_upper_cid_map[cid] = upper_cid
+	end
+
+	return lower_to_upper_cid_map
+end
+
+local function add_subblock(lower_data, lower_pos, upper_data)
+	local lower_to_upper_cid_map = make_lower_to_upper_cid_map(lower_data, upper_data)
+
+	local subblock_bit = (lower_pos.x % 16) +
+	                     (lower_pos.y % 16) * 16 +
+			     (lower_pos.z % 16) * 256
+
+	local upper_counts = upper_data.counts or {}
+	upper_data.counts = upper_counts
+
+	local subblock_indexes = upper_data.subblocks or {}
+	upper_data.subblocks = subblock_indexes
+
+	-- Clear out old subblock index flags
+	for cid,subblock_index in pairs(subblock_indexes) do
+		bitmap_index_clear(subblock_index, subblock_bit)
+	end
+
+	local lower_counts = lower_data.counts or {}
+	for lower_cid,lower_count in pairs(lower_counts) do
+		-- Add block counts from subblock into the upper block data
+		local upper_cid = lower_to_upper_cid_map[lower_cid]
+		local upper_count = upper_counts[upper_cid] or 0
+		upper_count = upper_count + lower_count
+		upper_counts[upper_cid] = upper_count
+
+		-- Flag the node type as existing in this subblock
+		local subblock_index = subblock_indexes[upper_cid] or {}
+		subblock_indexes[upper_cid] = subblock_index
+		bitmap_index_set(subblock_index, subblock_bit)
+	end
+end
+local function rem_subblock(lower_data, lower_pos, upper_data)
+	local lower_to_upper_cid_map = make_lower_to_upper_cid_map(lower_data, upper_data)
+
+	local upper_counts = upper_data.counts or {}
+	upper_data.counts = upper_counts
+
+	local lower_counts = lower_data.counts or {}
+	for lower_cid,lower_count in pairs(lower_counts) do
+		local upper_cid = lower_to_upper_cid_map[lower_cid]
+		local upper_count = upper_counts[upper_cid] or 0
+		upper_count = upper_count - lower_count
+		if upper_count < 0 then upper_count = 0 end
+		upper_counts[upper_cid] = upper_count
+	end
+end
+
+mod.register_indexer("node_counter",{
+	pre_process = function(data)
+		local pos = minetest.get_position_from_hash(data.map_block.hash)
+
+		-- L2 index (256x256x256 blocks)
+		local l2_block = mod.map_block(pos)
+		data.l2_data = storage:get_string("node_counter.l2"..vector.to_string(l2_block))
+
+		-- L3 index (4096*4096*4096 blocks)
+		local l3_block = mod.map_block(l2_block)
+		data.l3_data = storage:get_string("node_counter.l3"..vector.to_string(l3_block))
+
+		-- L4 (whole world, 65536x65536x65536 block)
+		data.l4_data = storage:get_string("node_counter.l4")
+
+	end,
+	index = function(data)
+		local min = data.map_block.min
+		local max = data.map_block.max
+		local area = VoxelArea:new{ MinEdge = data.emin, MaxEdge = data.emax }
+		local vm_data = data.vm:get_data()
+
+		local counts = {}
+		local under_air = {}
+
+		local cid_air = minetest.get_content_id("air")
+
+		local positions = {}
+
+		-- Count all nodes
+		for z = min.z,max.z do
+			for y = min.y,max.y do
+				for x = min.x,max.x do
+					local idx = area:index(x,y,z)
+					local cid = vm_data[idx]
+					counts[cid] = (counts[cid] or 0) + 1
+
+					local node_positions = positions[cid] or {}
+					positions[cid] = node_positions
+
+					if #node_positions < 32 then
+						node_positions[#node_positions + 1] = block_pos_hash(min, x, y, z)
+					end
+
+					--[[
+					if cid ~= cid_air and y < max.y - 1 and vm_data[area:index(x,y+1,z)] == cid_air then
+						local pos_hash = block_pos_hash(min, x, y, z)
+						local list = under_air[cid] or {}
+						under_air[cid] = list
+						list[#list + 1] = pos_hash
+					end
+					--]]
+				end
+			end
+		end
+
+		-- Convert counts to use node names instead of content ids
+		local cid_map = {}
+		for cid,count in pairs(counts) do
+			if not cid_map[cid] then
+				cid_map[cid] = minetest.get_name_from_content_id(cid)
+			end
+
+			-- Drop all position lists with more than 32 nodes in it
+			if count >= 32 then
+				positions[cid] = nil
+			end
+		end
+
+		-- This is our index
+		local index = {
+			counts = counts,
+			cid_map = cid_map,
+			positions = positions,
+			--under_air = under_air,
+		}
+
+		-- Now, let's build the heirarchial data
+		local pos = minetest.get_position_from_hash(data.map_block.hash)
+
+		local old_index = {}
+		if data.old_index ~= "" then
+			old_index = mod.unpack_index(data.old_index)
+		end
+		local l2_data = {}
+		local l2_pos = mod.map_block(pos)
+		if data.l2_data ~= "" then
+			l2_data = mod.unpack_index(data.l2_data)
+		end
+
+
+		local l3_data = {}
+		local l3_pos = mod.map_block(l2_pos)
+		if data.l3_data ~= "" then
+			l3_data = mod.unpack_index(data.l3_data)
+		end
+
+		local l4_data = {}
+		if data.l4_data ~= "" then
+			l4_data = mod.unpack_index(data.l4_data)
+		end
+
+		-- Update heirarchial indexes
+		rem_subblock(l3_data, l3_pos, l4_data)
+		rem_subblock(l2_data, l2_pos, l3_data)
+		rem_subblock(old_index, pos, l2_data)
+		add_subblock(index, pos, l2_data)
+		add_subblock(l2_data, l2_pos, l3_data)
+		add_subblock(l3_data, l3_pos, l4_data);
+
+		--[[
+		print(dump(index))
+		print(dump(l2_data))
+		print(dump(l3_data))
+		print(dump(l4_data))
+		--]]
+		return {
+			index = mod.pack_index(index),
+			l2_index = mod.pack_index(l2_data),
+			l3_index = mod.pack_index(l3_data),
+			l4_index = mod.pack_index(l4_data)
+		}
+	end,
+	post_process = function(map_block_hash, result)
+		local unpacked_result = mod.unpack_index(result)
+
+		local pos = minetest.get_position_from_hash(map_block_hash)
+		local l2_block = mod.map_block(pos)
+		local key = "node_counter.l2"..vector.to_string(l2_block)
+		storage:set_string(key, unpacked_result.l2_index)
+		mod.clear_cache(key)
+
+		local l3_block = mod.map_block(l2_block)
+		local key = "node_counter.l3"..vector.to_string(l3_block)
+		storage:set_string(key, unpacked_result.l3_index)
+		mod.clear_cache(key)
+
+		key = "node_counter.l4"
+		storage:set_string(key, unpacked_result.l4_index)
+		mod.clear_cache(key)
+
+		return unpacked_result.index
+	end,
+})
+
+if not mod.main_thread then return end
+
+local function closest_block_in_region(pos, min, max)
+	local closest = vector.new(pos)
+
+	if closest.x < min.x then
+		closest.x = min.x
+	elseif closest.x > max.x then
+		closest.x = max.x
+	end
+
+	if closest.y < min.y then
+		closest.y = min.y
+	elseif closest.y > max.y then
+		closest.y = max.y
+	end
+
+	if closest.z < min.z then
+		closest.z = min.z
+	elseif closest.z > max.z then
+		closest.z = max.z
+	end
+
+	return closest
+end
+local function distance_to_block(pos, min, max)
+	return vector.distance(pos, closest_block_in_region(pos, min, max))
+end
+
+local function expand_nodenames(nodenames, l4_index)
+	local nodes = {}
+	local groups = {}
+
+	for i = 1,#nodenames do
+		local name = nodenames[i]
+		if name:sub(1,6) == "group:" then
+			local group_name = name:sub(7)
+			groups[#groups + 1] = group_name
+		else
+			nodes[name] = true
+		end
+	end
+
+	local result = {}
+	local cid_from_nodename = {}
+	for cid,nodename in pairs(l4_index.cid_map) do
+		local nodedef = minetest.registered_nodes[nodename]
+		if nodes[nodename] then
+			result[#result + 1] = nodename
+			cid_from_nodename[nodename] = cid
+		elseif nodedef then
+			local node_groups = nodedef.groups
+			local hit = false
+			for j = 1,#groups do
+				if not hit and (node_groups[groups[j]] or 0) ~= 0 then
+					result[#result + 1] = nodename
+					cid_from_nodename[nodename] = cid
+					hit = true
+				end
+			end
+		end
+
+	end
+	return result,cid_from_nodename
+end
+function expand_subindex(nodenames, cid_from_nodename, subindex_name, index, size, cost_filter)
+	if not cid_from_nodename then
+		cid_from_nodename = {}
+		for cid,nodename in pairs(index.cid_map) do
+			cid_from_nodename[nodename] = cid
+		end
+	end
+
+	-- Get union of subnode bitmap index where these nodes are located
+	local union_index = {}
+	for i = 1,#nodenames do
+		local nodename = nodenames[i]
+		local cid = cid_from_nodename[nodename]
+		if cid then
+			local node_index  = index.subblocks[cid]
+			for subnode,part in pairs(node_index) do
+				local union_part = union_index[subnode] or 0
+				union_part = bit.bor(union_part, part)
+				union_index[subnode] = union_part
+			end
+		end
+	end
+
+	-- Build a cost-sorted candidate list from the list of subindexes that have the desired blocks
+	local candidates = {}
+	for k,v in pairs(union_index) do
+		for i=0,31 do
+			local bit = v % 2
+			v = (v - bit) / 2
+			if bit == 1 then
+				local offset = k * 32 + i
+				local x = offset % 16; offset = (offset - x)/16
+				local y = offset % 16; offset = (offset - y)/16
+				local z = offset % 16
+				local min = vector.multiply(vector.new(x,y,z), size)
+				local max = vector.offset(min, size-1, size-1, size-1)
+				local subindex = nil
+				if subindex_name then
+					subindex = string.format(subindex_name.."(%d, %d, %d)",x,y,z)
+				end
+				local cost = cost_filter(min, max)
+				if cost then
+					local idx = #candidates + 1
+					candidates[idx] = {cost, subindex, min, max}
+
+					-- Insersion sort according to cost so the lowest cost is at the end
+					if idx > 1 then
+						local a = candidates[idx]
+						local b = candidates[idx - 1]
+						while idx > 1 and a[1] > b[1] do
+							-- Swap
+							candidates[idx  ] = b
+							candidates[idx-1] = a
+							idx = idx - 1
+							b = candidates[idx - 1]
+						end
+					end
+				end
+			end
+		end
+	end
+	return candidates
+end
+
+-- Public-facing API
+function mod.find_nodes_in_area(pos1, pos2, nodenames, grouped)
+	local size = math.abs(pos1.x - pos2.x) * math.abs(pos1.y - pos2.y) * math.abs(pos1.z - pos2.z)
+	if size < MIN_SEARCH_SIZE then
+		return minetest.find_nodes_in_area(pos1, pos2, nodenames, grouped)
+	end
+
+	-- Start at the top-level index, then work down
+	local l4_index = mod.get_index("node_counter.l4")
+	print("l4_index="..dump(l4_index))
+
+	-- Expand nodenames to actual list of nodenames
+	local actual_nodenames = expand_nodenames(nodenames, l4_index)
+	if #nodenames == 0 then return {},{} end
+
+	-- No results
+	return {},{}
+end
+function mod.find_node_near(pos, radius, nodenames, search_center)
+	local size = radius * radius * radius
+	if size < MIN_SEARCH_SIZE then
+		return minetest.find_node_near(pos, radius, nodenames, search_center)
+	end
+
+	local function distance_filter(min, max)
+		local distance = distance_to_block(pos, min, max)
+		if distance > radius then return nil end
+		return distance
+	end
+
+	-- Start at the top-level index, then work down
+	local l4_index = mod.get_index("node_counter.l4")
+
+	-- Expand nodenames to actual list of nodenames, return if we can't match any indexed nodes
+	local nodenames,cid_from_nodename = expand_nodenames(nodenames, l4_index)
+	if #nodenames == 0 then return nil end
+
+	-- Expand out 3 more levels of subindexes, starting closest to pos
+	local l3_candidates = expand_subindex(nodenames, cid_from_nodename, "node_counter.l3", l4_index, 16*16*16, distance_filter )
+	local l3_idx = #l3_candidates
+	while l3_idx > 0 do
+		local l3_candidate = l3_candidates[l3_idx]
+		local l3_index = mod.get_index(l3_candidate[2])
+		local l2_candidates = expand_subindex(nodenames, nil, "node_counter.l2", l3_index, 16*16, distance_filter )
+		local l2_idx = #l2_candidates
+		while l2_idx > 0 do
+			local l2_candidate = l2_candidates[l2_idx]
+			local l2_index = mod.get_index(l2_candidate[2])
+			local l1_candidates = expand_subindex(nodenames, nil, "node_counter", l2_index, 16, distance_filter )
+			local l1_idx = #l1_candidates
+			while l1_idx > 0 do
+				local l1_candidate = l1_candidates[l1_idx]
+				local l1_index = mod.get_index(l1_candidate[2])
+				local block_min = l1_candidate[3]
+
+				local cid_from_nodename = {}
+				for cid,nodename in pairs(l1_index.cid_map) do
+					cid_from_nodename[nodename] = cid
+				end
+
+				-- Find closest node
+				local best_pos = nil
+				local best_dist = radius
+				for i = 1,#nodenames do
+					local cid = cid_from_nodename[nodenames[i]]
+					local cid_positions = l1_index.positions[cid] or {}
+					for j = 1,#cid_positions do
+						local cid_position = cid_positions[j]
+						local block_pos = get_block_pos_from_hash(block_min,cid_position)
+						local dist = vector.distance(block_pos, pos)
+						if dist < best_dist then
+							best_dist = dist
+							best_pos = block_pos
+						end
+					end
+				end
+
+				-- Because L3, L2 and L1 candidates  are all sorted by closest to pos, if we have
+				-- position at this point this will always be the closest indexed node inside the
+				-- provided radius
+				if best_pos then
+					return best_pos
+				end
+
+				-- remove the L1 candidate
+				l1_candidates[l1_idx] = nil
+				l1_idx = l1_idx - 1
+			end
+
+			-- remove the L2 candidate
+			l2_candidates[l2_idx] = nil
+			l2_idx = l2_idx - 1
+		end
+
+		-- Remove the L3 candidate
+		l3_candidates[l3_idx] = nil
+		l3_idx = l3_idx - 1
+	end
+
+	-- No result
+	return nil
+end
+

mods/CORE/vl_map_index/scheduler.lua

@@ -0,0 +1,84 @@
+local mod = vl_map_index
+
+-- Adapter for when no vl_scheduler is not available
+local periodic = {}
+local tasks = {}
+local functions = {}
+
+local scheduler = {
+	register_periodic_task = function(name,period,priority,func)
+		periodic[#periodic + 1] = {
+			func = func,
+			period = period,
+			time = 0
+		}
+		local time = 0
+		minetest.register_globalstep(function(dtime)
+			time = time + dtime
+			if time >= period then
+				func()
+				time = time - period
+			end
+		end)
+	end,
+	register_function = function(name, func, default_time, default_priority)
+		functions[name] = func
+	end,
+	add_task = function(name, time, priority, args)
+		local func = functions[name]
+		if not func then return end
+
+		local task = {
+			func = func,
+			time = time,
+			args = args,
+		}
+		tasks[#tasks + 1] = task
+	end,
+	add_cancelable_task = function(name, time, priority, args)
+		local func = functions[name]
+		if not func then return end
+
+		local task = {
+			func = func,
+			time = time,
+			args = args,
+		}
+		function task:cancel()
+			task.canceled = true
+		end
+		tasks[#tasks + 1] = task
+	end
+}
+mod.scheduler = scheduler
+
+minetest.register_globalstep(function(dtime)
+	-- Run periodic tasks
+	for i = 1,#periodic do
+		local task = periodic[i]
+		local time,period = task.time,task.period
+		time = time + dtime
+		if time >= period then
+			time = time - period
+			task.func()
+		end
+		task.time = time
+	end
+
+	-- Run tasks
+	local new_tasks = {}
+	for i = 1,#tasks do
+		local task = tasks[i]
+		local time = task.time
+		time = time - dtime
+		if time <= 0 then
+			if not task.canceled then
+				task.func(unpack(task.args))
+			end
+		else
+			task.time = time
+			new_tasks[#new_tasks + 1] = task
+		end
+	end
+	tasks = new_tasks
+end)

mods/ITEMS/mcl_lightning_rods/init.lua

@@ -93,7 +93,7 @@ minetest.register_node("mcl_lightning_rods:rod_powered", rod_def_a)
 
 
 lightning.register_on_strike(function(pos, pos2, objects)
-	local lr = minetest.find_node_near(pos, 128, { "group:attracts_lightning" }, true)
+	local lr = vl_map_index.find_node_near(pos, 128, { "group:attracts_lightning" }, true)
 
 	if lr then
 		local node = minetest.get_node(lr)