Serialize_lib: finish up and add atomic api
This commit is contained in:
orwell96
parent
544f8babb5
commit
02d845e83d
|
|
@ -0,0 +1,215 @@
|
|||
-- atomic.lua
|
||||
-- Utilities for transaction-like handling of serialized state files
|
||||
-- Also for multiple files that must be synchronous, as advtrains currently requires.
|
||||
|
||||
|
||||
-- Managing files and backups
|
||||
-- ==========================
|
||||
|
||||
--[[
|
||||
The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
|
||||
the write, so we have incomplete data. So, the following scheme is applied:
|
||||
Unix:
|
||||
1. writes to <filename>.new
|
||||
2. moves <filename>.new to <filename>, clobbering previous file
|
||||
Windows:
|
||||
1. writes to <filename>.new
|
||||
2. delete <filename>
|
||||
3. moves <filename>.new to <filename>
|
||||
|
||||
We count a new version of the state as "committed" after stage 2.
|
||||
|
||||
During loading, we apply the following order of precedence:
|
||||
1. <filename>
|
||||
2. <filename>.new (windows only, in case we were interrupted just before 3. when saving)
|
||||
|
||||
|
||||
All of these functions return either true on success or nil, error on error.
|
||||
]]--
|
||||
|
||||
local ser = serialize_lib.serialize
|
||||
|
||||
local windows_mode = false
|
||||
|
||||
-- == local functions ==
|
||||
|
||||
local function save_atomic_move_file(filename)
|
||||
--2. if windows mode, delete main file
|
||||
if windows_mode then
|
||||
local delsucc, err = os.remove(filename)
|
||||
if not delsucc then
|
||||
serialize_lib.log_error("Unable to delete old savefile '"..filename.."':")
|
||||
serialize_lib.log_error(err)
|
||||
return nil, err
|
||||
end
|
||||
end
|
||||
|
||||
--3. move file
|
||||
local mvsucc, err = os.rename(filename..".new", filename)
|
||||
if not mvsucc then
|
||||
if minetest.settings:get_bool("serialize_lib_no_auto_windows_mode") or windows_mode then
|
||||
serialize_lib.log_error("Unable to move '"..filename.."':")
|
||||
serialize_lib.log_error(err)
|
||||
return nil, err
|
||||
else
|
||||
-- enable windows mode and try again
|
||||
serialize_lib.log_info("Enabling Windows mode for atomic saving...")
|
||||
windows_mode = true
|
||||
return save_atomic_move_file(filename)
|
||||
end
|
||||
end
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
local function open_file_and_save_callback(callback, filename)
|
||||
local file, err = io.open(filename, "w")
|
||||
if not file then
|
||||
error("Failed opening file '"..filename.."' for write:\n"..err)
|
||||
end
|
||||
|
||||
callback(file)
|
||||
return true
|
||||
end
|
||||
|
||||
local function open_file_and_load_callback(filename, callback)
|
||||
local file, err = io.open(filename, "r")
|
||||
if not file then
|
||||
error("Failed opening file '"..filename.."' for read:\n"..err)
|
||||
end
|
||||
|
||||
return callback(file)
|
||||
end
|
||||
|
||||
-- == public functions ==
|
||||
|
||||
-- Load a saved state (according to comment above)
|
||||
-- if 'callback' is nil: reads serialized table.
|
||||
-- returns the read table, or nil,err on error
|
||||
-- if 'callback' is a function (signature func(file_handle) ):
|
||||
-- Counterpart to save_atomic with function argument. Opens the file and calls callback on it.
|
||||
-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
|
||||
-- The callback's first return value is returned by load_atomic
|
||||
function serialize_lib.load_atomic(filename, callback)
|
||||
|
||||
local cbfunc = callback or ser.read_from_fd
|
||||
|
||||
-- try <filename>
|
||||
local file, ret = io.open(filename, "r")
|
||||
if file then
|
||||
-- read the file using the callback
|
||||
local success
|
||||
success, ret = pcall(cbfunc, file)
|
||||
if success then
|
||||
return ret
|
||||
end
|
||||
end
|
||||
|
||||
if minetest.settings:get_bool("serialize_lib_strict_loading") then
|
||||
serialize_lib.save_lock = true
|
||||
error("Loading data from file '"..filename.."' failed:\n"
|
||||
..ret.."\nDisable Strict Loading to ignore.")
|
||||
end
|
||||
|
||||
serialize_lib.log_warn("Loading data from file '"..filename.."' failed, trying .new fallback:")
|
||||
serialize_lib.log_warn(ret)
|
||||
|
||||
-- try <filename>.new
|
||||
file, ret = io.open(filename..".new", "r")
|
||||
if file then
|
||||
-- read the file using the callback
|
||||
local success
|
||||
success, ret = pcall(cbfunc, file)
|
||||
if success then
|
||||
return ret
|
||||
end
|
||||
end
|
||||
|
||||
serialize_lib.log_error("Unable to load data from '"..filename..".new':")
|
||||
serialize_lib.log_error(ret)
|
||||
serialize_lib.log_error("Note: This message is normal when the mod is loaded the first time on this world.")
|
||||
|
||||
return nil, ret
|
||||
end
|
||||
|
||||
-- Save a file atomically (as described above)
|
||||
-- 'data' is the data to be saved (when a callback is used, this can be nil)
|
||||
-- if 'callback' is nil:
|
||||
-- data must be a table, and is serialized into the file
|
||||
-- if 'callback' is a function (signature func(data, file_handle) ):
|
||||
-- Opens the file and calls callback on it. The 'data' argument is the data passed to save_atomic().
|
||||
-- If the callback function throws an error, and strict loading is enabled, that error is propagated.
|
||||
-- The callback's first return value is returned by load_atomic
|
||||
-- Important: the callback must close the file in all cases!
|
||||
function serialize_lib.save_atomic(data, filename, callback, config)
|
||||
if serialize_lib.save_lock then
|
||||
serialize_lib.log_warn("Instructed to save '"..filename.."', but save lock is active!")
|
||||
return nil
|
||||
end
|
||||
|
||||
local cbfunc = callback or ser.write_to_fd
|
||||
|
||||
local file, ret = io.open(filename..".new", "w")
|
||||
if file then
|
||||
-- save the file using the callback
|
||||
local success
|
||||
success, ret = pcall(cbfunc, data, file)
|
||||
if success then
|
||||
return save_atomic_move_file(filename)
|
||||
end
|
||||
end
|
||||
serialize_lib.log_error("Unable to save data to '"..filename..".new':")
|
||||
serialize_lib.log_error(ret)
|
||||
return nil, ret
|
||||
end
|
||||
|
||||
|
||||
-- Saves multiple files synchronously. First writes all data to all <filename>.new files,
|
||||
-- then moves all files in quick succession to avoid inconsistent backups.
|
||||
-- parts_table is a table where the keys are used as part of the filename and the values
|
||||
-- are the respective data written to it.
|
||||
-- e.g. if parts_table={foo={...}, bar={...}}, then <filename_prefix>foo and <filename_prefix>bar are written out.
|
||||
-- if 'callbacks_table' is defined, it is consulted for callbacks the same way save_atomic does.
|
||||
-- example: if callbacks_table = {foo = func()...}, then the callback is used during writing of file 'foo' (but not for 'bar')
|
||||
-- Note however that you must at least insert a "true" in the parts_table if you don't use the data argument.
|
||||
-- Important: the callback must close the file in all cases!
|
||||
function serialize_lib.save_atomic_multiple(parts_table, filename_prefix, callbacks_table, config)
|
||||
if serialize_lib.save_lock then
|
||||
serialize_lib.log_warn("Instructed to save '"..filename_prefix.."' (multiple), but save lock is active!")
|
||||
return nil
|
||||
end
|
||||
|
||||
for subfile, data in pairs(parts_table) do
|
||||
local filename = filename_prefix..subfile
|
||||
local cbfunc = ser.write_to_fd
|
||||
if callbacks_table and callbacks_table[subfile] then
|
||||
cbfunc = callbacks_table[subfile]
|
||||
end
|
||||
|
||||
local success = false
|
||||
local file, ret = io.open(filename..".new", "w")
|
||||
if file then
|
||||
-- save the file using the callback
|
||||
success, ret = pcall(cbfunc, data, file, config)
|
||||
end
|
||||
|
||||
if not success then
|
||||
serialize_lib.log_error("Unable to save data to '"..filename..".new':")
|
||||
serialize_lib.log_error(ret)
|
||||
return nil, ret
|
||||
end
|
||||
end
|
||||
|
||||
local first_error
|
||||
for file, _ in pairs(parts_table) do
|
||||
local filename = filename_prefix..file
|
||||
local succ, err = save_atomic_move_file(filename)
|
||||
if not succ and not first_error then
|
||||
first_error = err
|
||||
end
|
||||
end
|
||||
|
||||
return not first_error, first_error -- either true,nil or nil,error
|
||||
end
|
||||
|
||||
|
||||
50
init.lua
50
init.lua
|
|
@ -30,63 +30,45 @@ Not all functions use all of the parameters, so you can simplify your config som
|
|||
|
||||
|
||||
function serialize_lib.log_error(text)
|
||||
minetest.log("error", "[serialize_lib] "..text)
|
||||
minetest.log("error", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
|
||||
end
|
||||
function serialize_lib.log_warn(text)
|
||||
minetest.log("warning", "[serialize_lib] "..text)
|
||||
minetest.log("warning", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
|
||||
end
|
||||
function serialize_lib.log_info(text)
|
||||
minetest.log("action", "[serialize_lib] "..text)
|
||||
minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?").."): "..(text or "<nil>"))
|
||||
end
|
||||
function serialize_lib.log_debug(text)
|
||||
minetest.log("action", "[serialize_lib](debug) "..text)
|
||||
minetest.log("action", "[serialize_lib] ("..(minetest.get_current_modname() or "?")..") DEBUG: "..(text or "<nil>"))
|
||||
end
|
||||
|
||||
-- basic serialization/deserialization
|
||||
-- ===================================
|
||||
|
||||
local ser = dofile("serialize.lua")
|
||||
local mp = minetest.get_modpath(minetest.get_current_modname())
|
||||
serialize_lib.serialize = dofile(mp.."/serialize.lua")
|
||||
dofile(mp.."/atomic.lua")
|
||||
|
||||
local ser = serialize_lib.serialize
|
||||
|
||||
-- Opens the passed filename, and returns deserialized table
|
||||
-- When an error occurs, logs an error and returns false
|
||||
function serialize_lib.read_table_from_file(filename)
|
||||
local succ, err = pcall(ser.read_from_file, filename)
|
||||
local succ, ret = pcall(ser.read_from_file, filename)
|
||||
if not succ then
|
||||
serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
|
||||
serialize_lib.log_error(ret)
|
||||
end
|
||||
return succ
|
||||
return ret
|
||||
end
|
||||
|
||||
-- Writes table into file
|
||||
-- When an error occurs, logs an error and returns false
|
||||
function serialize_lib.write_table_to_file(filename)
|
||||
local succ, err = pcall(ser.write_to_file, filename)
|
||||
function serialize_lib.write_table_to_file(root_table, filename)
|
||||
local succ, ret = pcall(ser.write_to_file, root_table, filename)
|
||||
if not succ then
|
||||
serialize_lib.log_error("Mod '"..minetest.get_current_modname().."': "..err)
|
||||
serialize_lib.log_error(ret)
|
||||
end
|
||||
return succ
|
||||
return ret
|
||||
end
|
||||
|
||||
-- Managing files and backups
|
||||
-- ==========================
|
||||
|
||||
--[[
|
||||
The plain scheme just overwrites the file in place. This however poses problems when we are interrupted right within
|
||||
the write, so we have incomplete data. So, the following scheme is applied:
|
||||
1. writes to <filename>.new (if .new already exists, try to complete the moving first)
|
||||
2. moves <filename> to <filename>.old, possibly overwriting an existing file (special windows behavior)
|
||||
3. moves <filename>.new to <filename>
|
||||
|
||||
During loading, we apply the following order of precedence:
|
||||
1. <filename>.new
|
||||
2. <filename>
|
||||
3. <filename>.old
|
||||
|
||||
Normal case: <filename> and <filename>.old exist, loading <filename>
|
||||
Interrupted during write: .new is damaged, loads last regular state
|
||||
Interrupted during the move operations: either <filename>.new or <filename> represents the latest state
|
||||
Other corruption: at least the .old state may still be present
|
||||
|
||||
]]--
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ function write_table(t, file, config)
|
|||
|
||||
if istable then
|
||||
vs = "T"
|
||||
if config.skip_empty_tables then
|
||||
if config and config.skip_empty_tables then
|
||||
writeit = not table_is_empty(value)
|
||||
end
|
||||
else
|
||||
|
|
@ -75,6 +75,7 @@ end
|
|||
|
||||
function value_to_string(t)
|
||||
if type(t)=="table" then
|
||||
file:close()
|
||||
error("Can not serialize a table in the key position!")
|
||||
elseif type(t)=="boolean" then
|
||||
if t then
|
||||
|
|
@ -87,6 +88,7 @@ function value_to_string(t)
|
|||
elseif type(t)=="string" then
|
||||
return "S"..escape_chars(t)
|
||||
else
|
||||
file:close()
|
||||
error("Can not serialize '"..type(t).."' type!")
|
||||
end
|
||||
return str
|
||||
|
|
@ -108,6 +110,7 @@ function read_table(t, file)
|
|||
while true do
|
||||
line = file:read("*l")
|
||||
if not line then
|
||||
file:close()
|
||||
error("Unexpected EOF or read error!")
|
||||
end
|
||||
|
||||
|
|
@ -117,6 +120,7 @@ function read_table(t, file)
|
|||
end
|
||||
ks, vs = string.match(line, "^(.+[^&]):(.+)$")
|
||||
if not ks or not vs then
|
||||
file:close()
|
||||
error("Unable to parse line: '"..line.."'!")
|
||||
end
|
||||
kv = string_to_value(ks)
|
||||
|
|
@ -137,6 +141,7 @@ function string_to_value(str, table_allow)
|
|||
if table_allow then
|
||||
return {}, true
|
||||
else
|
||||
file:close()
|
||||
error("Table not allowed in key component!")
|
||||
end
|
||||
elseif first=="N" then
|
||||
|
|
@ -144,6 +149,7 @@ function string_to_value(str, table_allow)
|
|||
if num then
|
||||
return num
|
||||
else
|
||||
file:close()
|
||||
error("Unable to parse number: '"..rest.."'!")
|
||||
end
|
||||
elseif first=="B" then
|
||||
|
|
@ -152,11 +158,13 @@ function string_to_value(str, table_allow)
|
|||
elseif rest=="1" then
|
||||
return true
|
||||
else
|
||||
file:close()
|
||||
error("Unable to parse boolean: '"..rest.."'!")
|
||||
end
|
||||
elseif first=="S" then
|
||||
return unescape_chars(rest)
|
||||
else
|
||||
file:close()
|
||||
error("Unknown literal type '"..first.."' for literal '"..str.."'!")
|
||||
end
|
||||
end
|
||||
|
|
@ -177,20 +185,20 @@ config = {
|
|||
}
|
||||
]]
|
||||
|
||||
-- Writes the passed table into the passed file descriptor, and closes the file afterwards
|
||||
-- Writes the passed table into the passed file descriptor, and closes the file
|
||||
local function write_to_fd(root_table, file, config)
|
||||
file:write("LUA_SER v=1\n")
|
||||
write_table(root_table, file, config)
|
||||
file:write("E\nEND_SER\n")
|
||||
file:close()
|
||||
end
|
||||
|
||||
-- Reads the file contents from the passed file descriptor and returns the table on success
|
||||
-- Throws errors when something is wrong.
|
||||
-- Throws errors when something is wrong. Closes the file.
|
||||
-- config: see above
|
||||
local function read_from_fd(file)
|
||||
local first_line = file:read("*l")
|
||||
if first_line ~= "LUA_SER v=1" then
|
||||
file:close()
|
||||
error("Expected header, got '"..first_line.."' instead!")
|
||||
end
|
||||
local t = {}
|
||||
|
|
@ -209,7 +217,7 @@ function write_to_file(root_table, filename, config)
|
|||
-- try opening the file
|
||||
local file, err = io.open(filename, "w")
|
||||
if not file then
|
||||
error("Failed opening file '"..filename.."' for write: "..err)
|
||||
error("Failed opening file '"..filename.."' for write:\n"..err)
|
||||
end
|
||||
|
||||
write_to_fd(root_table, file, config)
|
||||
|
|
@ -221,7 +229,7 @@ function read_from_file(filename)
|
|||
-- try opening the file
|
||||
local file, err = io.open(filename, "r")
|
||||
if not file then
|
||||
error("Failed opening file '"..filename.."' for read: "..err)
|
||||
error("Failed opening file '"..filename.."' for read:\n"..err)
|
||||
end
|
||||
|
||||
return read_from_fd(file)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
# Enable strict file loading mode
|
||||
# If enabled, if any error occurs during loading of a file using the 'atomic' API,
|
||||
# an error is thrown. You probably need to disable this option for initial loading after
|
||||
# creating the world.
|
||||
serialize_lib_strict_loading (Strict loading) bool false
|
||||
|
||||
# Do not automatically switch to "Windows mode" when saving atomically
|
||||
# Normally, when renaming <filename>.new to <filename> fails, serialize_lib automatically
|
||||
# switches to a mode where it deletes <filename> prior to moving. Enable this option to prevent
|
||||
# this behavior and abort saving instead.
|
||||
serialize_lib_no_auto_windows_mode (No automatic Windows Mode) bool false
|
||||
|
||||
Loading…
Reference in New Issue