better map colors

mods/ITEMS/mcl_maps/init.lua

@@ -30,11 +30,6 @@ local function load_json_file(name)
 end
 
 local texture_colors = load_json_file("colors")
-local palettes_grass = load_json_file("palettes_grass")
-local palettes_foliage = load_json_file("palettes_foliage")
-local palettes_water = load_json_file("palettes_water")
-
-local color_cache = {}
 
 local creating_maps = {}
 local loaded_maps = {}
@@ -66,80 +61,48 @@ function mcl_maps.create_map(pos)
 		local param2data = vm:get_param2_data()
 		local area = VoxelArea:new({ MinEdge = emin, MaxEdge = emax })
 		local pixels = {}
-		local last_heightmap
-		for x = 1, 128 do
-			local map_x = minp.x - 1 + x
-			local heightmap = {}
 		for z = 1, 128 do
 			local map_z = minp.z - 1 + z
-				local color, height
+			local last_height
+			for x = 1, 128 do
+				local map_x = minp.x - 1 + x
+				local cagg, alpha, height = { 0, 0, 0 }, 0
 				for map_y = maxp.y, minp.y, -1 do
 					local index = area:index(map_x, map_y, map_z)
 					local c_id = data[index]
 					if c_id ~= c_air then
-						color = color_cache[c_id]
-						if color == nil then
-							local nodename = minetest.get_name_from_content_id(c_id)
-							local def = minetest.registered_nodes[nodename]
-							if def then
-								local texture
-								if def.palette then
-									texture = def.palette
-								elseif def.tiles then
-									texture = def.tiles[1]
-									if type(texture) == "table" then
-										texture = texture.name
-									end
-								end
-								if texture then
-									texture = texture:match("([^=^%^]-([^.]+))$"):split("^")[1]
-								end
-								if def.palette == "mcl_core_palette_grass.png" then
-									local palette = palettes_grass[texture]
-									color = palette and { palette = palette }
-								elseif def.palette == "mcl_core_palette_foliage.png" then
-									local palette = palettes_foliage[texture]
-									color = palette and { palette = palette }
-								elseif def.palette == "mcl_core_palette_water.png" then
-									local palette = palettes_water[texture]
-									color = palette and { palette = palette }
-								else
-									color = texture_colors[texture]
-								end
-							end
+						local color = texture_colors[minetest.get_name_from_content_id(c_id)]
+						-- use param2 if available:
+						if color and type(color[1]) == "table" then
+							color = color[param2data[index] + 1] or color[1]
 						end
+						if color then
+							local a = (color[4] or 255) / 255
+							local f = a * (1 - alpha)
+							cagg[1] = cagg[1] + f * color[1]
+							cagg[2] = cagg[2] + f * color[2]
+							cagg[3] = cagg[3] + f * color[3]
+							alpha = alpha + f
 
-						if color and color.palette then
-							color = color.palette[param2data[index] + 1]
-						else
-							color_cache[c_id] = color or false
-						end
-
-						if color and last_heightmap then
-							local last_height = last_heightmap[z]
-							if last_height < map_y then
-								color = {
-									math.min(255, color[1] + 16),
-									math.min(255, color[2] + 16),
-									math.min(255, color[3] + 16),
-								}
-							elseif last_height > map_y then
-								color = {
-									math.max(0, color[1] - 16),
-									math.max(0, color[2] - 16),
-									math.max(0, color[3] - 16),
+							-- ground estimate with transparent blocks
+							if alpha > 0.70 and not height then height = map_y end
+							-- adjust color to give a 3d effect
+							if alpha >= 0.99 and last_height and height then
+								local dheight = math.min(math.max((height - last_height) * 8, -32), 32)
+								cagg = {
+									math.max(0, math.min(255, cagg[1] + dheight)),
+									math.max(0, math.min(255, cagg[2] + dheight)),
+									math.max(0, math.min(255, cagg[3] + dheight)),
 								}
 							end
-						end
-						height = map_y
-						break
+							if alpha >= 0.99 then break end
 						end
 					end
-				heightmap[z] = height or minp.y
+				end
+				last_height = height
 				pixels[z] = pixels[z] or {}
-				pixels[z][x] = color or { 0, 0, 0 }
+				pixels[z][x] = cagg or { 0, 0, 0 }
 			end
-			last_heightmap = heightmap
 		end
 		tga_encoder.image(pixels):save(map_textures_path .. "mcl_maps_map_texture_" .. id .. ".tga")
 		creating_maps[id] = nil

mods/ITEMS/mcl_maps/palettes_foliage.json

@@ -1 +0,0 @@
-{"mcl_core_palette_foliage.png": [[86, 164, 117], [109, 196, 117], [118, 177, 120], [159, 193, 114], [159, 193, 114], [74, 107, 58], [94, 190, 107], [94, 190, 107], [222, 188, 101], [90, 197, 87], [35, 175, 105], [92, 182, 119], [93, 181, 76], [93, 181, 76], [82, 153, 81], [91, 177, 85], [86, 164, 117], [94, 190, 107]]}

mods/ITEMS/mcl_maps/palettes_grass.json

@@ -1 +0,0 @@
-{"mcl_core_palette_grass.png": [[109, 196, 117], [159, 193, 114], [118, 177, 120], [118, 177, 120], [107, 186, 107], [118, 177, 120], [92, 182, 119], [92, 182, 119], [92, 182, 119], [92, 182, 119], [118, 177, 120], [109, 196, 117], [35, 175, 105], [94, 190, 107], [94, 190, 107], [94, 190, 107], [94, 190, 107], [159, 193, 114], [76, 176, 84], [164, 150, 110], [164, 150, 110], [164, 150, 110], [164, 150, 110], [159, 193, 114], [93, 181, 76], [93, 181, 76], [93, 181, 76], [93, 181, 76], [76, 118, 60], [94, 190, 107]]}

mods/ITEMS/mcl_maps/palettes_water.json

@@ -1 +0,0 @@
-{"mcl_core_palette_water.png": [[63, 118, 228], [82, 121, 179], [66, 149, 235], [65, 174, 233], [62, 104, 221], [60, 93, 215], [46, 100, 218], [61, 120, 181]]}

mods/MAPGEN/mcl_mapgen_core/api.lua

@@ -3,6 +3,7 @@ local log_timing = minetest.settings:get_bool("mcl_logging_mapgen_timing", false
 
 local registered_generators = {}
 local lvm, nodes, param2 = 0, 0, 0
+local lvm_buffer, lvm_buffer2 = {}, {}
 
 local seed = minetest.get_mapgen_setting("seed")
 
@@ -11,8 +12,8 @@ minetest.register_on_generated(function(minp, maxp, blockseed)
 	if lvm > 0 then
 		local vm, emin, emax = minetest.get_mapgen_object("voxelmanip")
 		local area = VoxelArea(emin, emax)
-		local data = vm:get_data()
-		local data2 = param2 > 0 and vm:get_param2_data()
+		local data = vm:get_data(lvm_buffer)
+		local data2 = param2 > 0 and vm:get_param2_data(lvm_buffer2)
 		if log_timing then
 			minetest.log("action", string.format("[mcl_mapgen_core] %-20s %s ... %s %8.2fms", "get_data", minetest.pos_to_string(minp), minetest.pos_to_string(maxp), (os.clock() - t1)*1000))
 		end

mods/MAPGEN/tsm_railcorridors/init.lua

@@ -398,7 +398,7 @@ end
 -- This is a workaround thanks to the fact that minetest.add_entity is unreliable as fuck
 -- See: https://github.com/minetest/minetest/issues/4759
 -- FIXME: Kill this horrible hack with fire as soon you can.
-local RecheckCartHack = function() end
+local RecheckCartHack = nil
 if not minetest.features.random_state_restore then -- proxy for minetest > 5.9.0, this feature will not be removed
 RecheckCartHack = function(params)
 	local pos = params[1]
@@ -946,7 +946,7 @@ local function spawn_carts()
 			-- TODO: Move callback function to this position when the
 			-- minetest.add_entity bug has been fixed (supposedly in 5.9.0?)
 			if RecheckCartHack then
-				minetest.after(3, RecheckCartHack, {cpos, cart_id})
+				minetest.after(1, RecheckCartHack, {cpos, cart_id})
 			else
 				tsm_railcorridors.on_construct_cart(cpos, obj, pr_carts)
 			end

tools/minetestmapper/dumpnodes/init.lua

@@ -1,11 +1,3 @@
-local function get_tile(tiles, n)
-	local tile = tiles[n]
-	if type(tile) == 'table' then
-		return tile.name or tile.image
-	end
-	return tile
-end
-
 local function pairs_s(dict)
 	local keys = {}
 	for k in pairs(dict) do
@@ -44,16 +36,20 @@ minetest.register_chatcommand("dumpnodes", {
 				if tiles == nil or nd.drawtype == 'airlike' then
 					print("ignored(2): " .. nn)
 				else
-					local tex = get_tile(tiles, 1)
-					tex = (tex .. '^'):match('%(*(.-)%)*^') -- strip modifiers
-					if tex:find("[combine", 1, true) then
-						tex = tex:match('.-=([^:]-)') -- extract first texture
+					local tex, opts = nil, ""
+					for i = 1, #tiles do
+						local tile = tiles[i]
+						tex = type(tile) == 'table' and (tile.name or tile.image) or tile
+						if tex ~= "blank.png" then break end
 					end
-					local opts = ""
+					if tex then
 						if nd.paramtype2 and nd.paramtype2:sub(1,5) == "color" and nd.palette ~= "" then
 							opts = " " .. nd.paramtype2 .. " " .. nd.palette
+						elseif nd.color and nd.color ~= "" then
+							opts = " " .. nd.color
 						end
 						out:write(nn .. ' ' .. tex .. opts .. '\n')
+					end
 					n = n + 1
 				end
 			end

tools/minetestmapper/generate_colorstxt.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
-import sys
-import os.path
-import getopt
-import re
+import sys, os.path, getopt, re, json
 from math import sqrt
 try:
-	from PIL import Image
+	from pyparsing import CharsNotIn, Suppress, infix_notation, opAssoc, ZeroOrMore
+except:
+	print("Could not load pyparser library, install 'pyparsing'!", file=sys.stderr)
+	exit(1)
+try:
+	from PIL import Image, ImageColor, ImageChops, ImageEnhance, ImageMath
 except:
 	print("Could not load image routines, install PIL ('pillow' on pypi)!", file=sys.stderr)
 	exit(1)
@@ -18,8 +20,11 @@ except:
 # 3) Run this script and poin it to the installation path of the game using -g,
 #    the path(s) where mods are stored using -m and the nodes.txt in your world folder.
 #    Example command line:
-#      ./util/generate_colorstxt.py --game /usr/share/minetest/games/minetest_game \
-#        -m ~/.minetest/mods ~/.minetest/worlds/my_world/nodes.txt
+#      ./util/generate_colorstxt.py \
+#        --game /usr/share/minetest/games/minetest_game \
+#        --mods ~/.minetest/mods \
+#        --mods /usr/share/minetest/textures \
+#        ~/.minetest/worlds/my_world/nodes.txt
 # 4) Copy the resulting colors.txt file to your world folder or to any other places
 #    and use it with minetestmapper's --colors option.
 ###########
@@ -58,8 +63,7 @@ def collect_files(path):
 			if not f in textures:
 				textures[f] = os.path.join(dirpath, f)
 
-def average_color(filename, color2):
-	inp = Image.open(filename).convert('RGBA')
+def average_color(name, inp, color2):
 	data = inp.load()
 
 	c, w = [0, 0, 0], 0
@@ -74,8 +78,8 @@ def average_color(filename, color2):
 			w = w + a
 
 	if w == 0:
-		print(f"didn't find color for '{os.path.basename(filename)}'", file=sys.stderr)
-		return "0 0 0"
+		print(f"didn't find color for '{name}'", file=sys.stderr)
+		return None
 	c0, c1, c2 = c[0] / w, c[1] / w, c[2] / w
 	if color2: # param2 blending
 		c0, c1, c2 = c0 * color2[0] / 255., c1 * color2[1] / 255., c2 * color2[2] / 255.
@@ -92,16 +96,30 @@ def average_color(filename, color2):
 			a = max(a, a2)
 
 	if a > 0 and a < 190:
-		return "%d %d %d %d" % (c0, c1, c2, a)
-	return "%d %d %d" % (c0, c1, c2)
+		return tuple(int(round(x)) for x in (c0, c1, c2, a))
+	return tuple(int(round(x)) for x in (c0, c1, c2))
+
+_param2_cache = dict()
+def get_param2_colors(filename):
+	if not filename: return None
+	cols = _param2_cache.get(filename)
+	if not cols and filename in textures:
+		inp = Image.open(textures[filename]).convert('RGBA')
+		data = inp.load()
+		cols = []
+		for y in range(inp.size[1]):
+			for x in range(inp.size[0]):
+				col = data[x, y]
+				if col[3] == 0: break
+				assert len(cols) == x + y * inp.size[0]
+				cols.append((col[0], col[1], col[2])) # copy
+		_param2_cache[filename] = cols
+	return cols
 
 def get_param2_color(filename, param2):
 	if not filename: return None
-	inp = Image.open(filename).convert('RGBA')
-	data = inp.load()
-	x, y = param2 % inp.size[0], param2 // inp.size[0]
-	col = data[y, x]
-	return col[0], col[1], col[2] # copy
+	cols = get_param2_colors(filename)
+	return cols[param2] if cols else None
 
 def apply_sed(line, exprs):
 	for expr in exprs:
@@ -118,6 +136,170 @@ def apply_sed(line, exprs):
 	return line
 #
 
+# global texture cache
+textures = {}
+
+##### Texture parser
+
+# Pure image load
+class Filename:
+	def __init__(self, tokens):
+		self.fn = tokens[0]
+	
+	def gen(self, prev=None):
+		if not self.fn in textures:
+			raise FileNotFoundError(self.fn)
+		im = Image.open(textures[self.fn]).convert('RGBA')
+		if not prev: return im
+		prev.alpha_composite(im)
+		return prev
+
+	def pprint(self):
+		print("Load " + self.fn)
+
+# Filter operations - todo: split them in the parser already?
+class Filter:
+	_combinere = re.compile(r"(-?\d+),(-?\d+)=(.*)")
+
+	def __init__(self, tokens):
+		self.fname = tokens[0]
+		self.opts = tokens[1:]
+
+	def gen(self, prev=None):
+		# complex image loading filter, the most important one
+		if self.fname in ["combine"]:
+			assert prev is None
+			#print(self.fname, self.opts)
+			w, h = map(int, self.opts[0].split("x"))
+			im = Image.new("RGBA", (w,h), (255,255,0,0))
+			for blit in self.opts[1:]:
+				blit = Filter._combinere.match(blit).groups()
+				x, y, bfn = int(blit[0]), int(blit[1]), blit[2]
+				if not bfn in textures:
+					print("Skipping missing texture:", bfn, file=sys.stderr)
+					return im
+				t = Image.open(textures[bfn]).convert('RGBA')
+				im.alpha_composite(t, dest=(x,y))
+			return im
+		elif self.fname == "transformFX":
+			return prev.transpose(Image.Transpose.FLIP_LEFT_RIGHT)
+		elif self.fname == "transformFY":
+			return prev.transpose(Image.Transpose.FLIP_TOP_BOTTOM)
+		elif self.fname == "transformR90":
+			return prev.transpose(Image.Transpose.ROTATE_90)
+		elif self.fname == "transformR180":
+			return prev.transpose(Image.Transpose.ROTATE_180)
+		elif self.fname == "transformR270":
+			return prev.transpose(Image.Transpose.ROTATE_270)
+		elif self.fname == "opacity":
+			#print(self.fname, self.opts)
+			f = int(self.opts[0]) / 255.
+			bands = prev.split()
+			bands[3].point(lambda x: x * f)
+			return Image.merge('RGBA', bands)
+		elif self.fname == "noalpha":
+			prev.putalpha(255)
+			return prev
+		elif self.fname == "multiply":
+			#print(self.fname, self.opts)
+			col = ImageColor.getrgb(self.opts[0])
+			im = Image.new("RGB", prev.size, col)
+			bands = prev.split()
+			im = ImageChops.multiply(im, Image.merge('RGB', bands[:3]))
+			im.putalpha(bands[3])
+			return im
+		elif self.fname == "brighten":
+			im = Image.new("RGB", prev.size, (255,255,255))
+			bands = prev.split()
+			im = Image.blend(im, Image.merge('RGB', bands[:3]), 0.5)
+			im.putalpha(bands[3])
+			return im
+		elif self.fname == "hsl":
+			#print(self.fname, self.opts)
+			assert self.opts[0] == "0", "Color shifts are currently not implemented." ## TODO
+			assert len(self.opts) == 2, "Only saturation is currently supported." ## TODO
+			f = int(self.opts[1])
+			return ImageEnhance.Color(prev).enhance(f/100. + 1)
+		elif self.fname == "colorize":
+			# Needs testing.
+			# print(self.fname, self.opts, prev.size)
+			col = ImageColor.getrgb(self.opts[0])
+			if len(self.opts) == 1:
+				im = Image.new("RGB", prev.size, col)
+				mask = prev.getchannel("A")
+				mask.point(lambda x: 255 if x > 0 else 0)
+				im.putalpha(mask)
+				return im
+			elif self.opts[1] == "alpha":
+				im = Image.new("RGB", prev.size, col)
+				im.putalpha(prev.getchannel("A"))
+				return im
+			else:
+				f = int(self.opts[1]) / 255.
+				im = Image.new("RGBA", prev.size, col)
+				mask = prev.getchannel("A")
+				mask.point(lambda x: 255 if x > 0 else 0)
+				im = Image.blend(prev, im, f)
+				im.putalpha(mask)
+				assert im.has_transparency_data
+				return im
+		elif self.fname in ["resize"]:
+			#print(self.fname, self.opts)
+			w, h = map(int, self.opts[0].split("x"))
+			return prev.resize((w,h))
+		elif self.fname in ["mask"]:
+			#print(self.fname, self.opts)
+			# bitwise AND, very odd operation
+			mfn = self.opts[0]
+			if not mfn in textures:
+				print("Skipping missing texture:", mfn, file=sys.stderr)
+				return prev
+			m = Image.open(textures[mfn]).convert('RGBA')
+			return Image.merge('RGBA', [ImageMath.unsafe_eval("a&b", a=a, b=b).convert("L") for a,b in zip(prev.split(), m.split())])
+		elif self.fname in ["lowpart"]:
+			#print(self.fname, self.opts)
+			f = int(self.opts[0]) / 100
+			t = int(prev.size[1] * f)
+			return prev.crop((0, t, prev.size[0], prev.size[1]))
+		elif self.fname in ["verticalframe"]:
+			#print(self.fname, self.opts)
+			vdiv, idx = int(self.opts[0]), int(self.opts[1])
+			h = prev.size[1] // vdiv
+			return prev.crop((0, h * idx, prev.size[0], h * (idx + 1)))
+		print("Texture filter", self.fname, *self.opts, "not implemented yet.", file=sys.stderr)
+
+	def pprint(self):
+		print(self.fname, *self.opts)
+
+class Overlay:
+	def __init__(self, tokens):
+		self.overlays = tokens[0]
+	
+	def gen(self, prev=None):
+		cur = prev
+		for o in self.overlays:
+			cur = o.gen(cur)
+		return cur
+
+	def pprint(self):
+		for o in self.overlays:
+			o.pprint()
+
+
+# not sure how we would define escapes for filenames with ^ : or backslash
+filt = (Suppress("[") + CharsNotIn("^[():")("name") + ZeroOrMore(Suppress(":") + CharsNotIn("^[():")("opt*")))("filter*")
+filt.set_parse_action(Filter)
+fname = CharsNotIn("^():\\")("filename*")
+fname.set_parse_action(Filename)
+parser = infix_notation(filt ^ fname, lpar=Suppress('('), rpar=Suppress(')'),
+	op_list=[(Suppress("^"), 2, opAssoc.LEFT, Overlay)])
+
+#e = expr.parse_string("([combine:16x16:0,14=mcl_farming_pumpkin_stem_disconnected.png^[test)^[colorize:#2E9D2E:127^foobar.png", parse_all=None)
+#print(e)
+#e[0].pprint()
+
+#sys.exit(1)
+
 try:
 	opts, args = getopt.getopt(sys.argv[1:], "hg:m:", ["help", "game=", "mods=", "replace="])
 except getopt.GetoptError as e:
@@ -129,6 +311,7 @@ if ('-h', '') in opts or ('--help', '') in opts:
 
 input_file = "./nodes.txt"
 output_file = "./colors.txt"
+json_file = "./colors.json"
 texturepaths = []
 
 try:
@@ -170,41 +353,68 @@ if not os.path.exists(input_file) or os.path.isdir(input_file):
 	print(f"Input file '{input_file}' does not exist.", file=sys.stderr)
 	exit(1)
 
-#
-
+# Find textures
 print(f"Collecting textures from {len(texturepaths)} path(s)... ", end="", flush=True)
-textures = {}
 for path in texturepaths:
 	collect_files(path)
 print("done", len(textures), "files")
 
 print("Processing nodes...")
+cmap = dict()
 fin = open(input_file, 'r')
-fout = open(output_file, 'w')
-n = 0
 for line in fin:
 	line = line.rstrip('\r\n')
 	if not line or line[0] == '#':
-		fout.write(line + '\n')
+		#fout.write(line + '\n')
 		continue
 	line = line.split(" ")
 	node, tex = line[0], line[1]
 	if not tex or tex == "blank.png":
 		continue
-	if tex not in textures:
+	im = None
+	if "^" in tex or "[" in tex:
+		#print(node, tex)
+		im = parser.parse_string(tex)[0].gen()
+		#assert not "/" in node
+		#im.save(os.path.join("/tmp/test",node+".png"))
+	elif tex not in textures:
 		print(f"skip {node} texture {tex} not found")
 		continue
+	else:
+		im = Image.open(textures[tex]).convert("RGBA")
 	# TODO: full param2 support
 	color2 = None
-	if len(line) > 3 and line[2].startswith("color"):
-		color2 = get_param2_color(textures.get(line[3]), 0)
-	color = average_color(textures[tex], color2)
+	if len(line) == 3 and line[2].startswith("#"):
+		color2 = ImageColor.getrgb(line[2])
+	elif len(line) > 3 and line[2].startswith("color"): # FIXME: colorwallmounted etc.
+		tints = get_param2_colors(line[3])
+		if tints:
+			cmap[node] = [average_color(node+" "+tex, im, v) for v in tints]
+			continue
+		print("Unsupported:", *line)
+	elif len(line) > 2:
+		print("Unsupported:", *line[2:])
+	color = average_color(node+" "+tex, im, color2)
+	cmap[node] = color
+fin.close()
+
+n = 0
+fout = open(output_file, 'w')
+prefix = ""
+for node, color in sorted(cmap.items()):
+	if not prefix or not node.startswith(prefix):
+		prefix = node.split(":")[0] + ":"
+		fout.write("\n# " + prefix[:-1] + "\n")
+	if not color: # Try stripping off last _postfix
+		color = cmap.get(node.rsplit("_", 1)[0])
+	if not color: continue
+	if isinstance(color, list) and not isinstance(color[0], int): color = color[0] # needs minetestmapper support first
+	color = " ".join(str(x) for x in color)
 	line = f"{node} {color}"
-	#print(f"ok {node}")
 	line = apply_sed(line, REPLACEMENTS)
 	if line:
 		fout.write(line + '\n')
 		n += 1
-fin.close()
 fout.close()
+json.dump(cmap, open(json_file, "w"))
 print(f"Done, {n} entries written.")