Cache textures by checksum

src/CMakeLists.txt

@@ -238,6 +238,7 @@ set(minetest_SRCS
 	guiCreateWorld.cpp
 	guiConfirmMenu.cpp
 	client.cpp
+	filecache.cpp
 	tile.cpp
 	game.cpp
 	main.cpp

src/client.cpp

@@ -38,6 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "sha1.h"
 #include "base64.h"
 #include "clientmap.h"
+#include "filecache.h"
 #include "sound.h"
 
 static std::string getTextureCacheDir()
@@ -255,6 +256,7 @@ Client::Client(
 	m_map_seed(0),
 	m_password(password),
 	m_access_denied(false),
+	m_texture_cache(getTextureCacheDir()),
 	m_texture_receive_progress(0),
 	m_textures_received(false),
 	m_itemdef_received(false),
@@ -1412,7 +1414,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 			//read texture from cache
 			std::string name = deSerializeString(is);
 			std::string sha1_texture = deSerializeString(is);
-			
+
 			// if name contains illegal characters, ignore the texture
 			if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
 				errorstream<<"Client: ignoring illegal texture name "
@@ -1420,74 +1422,50 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 				continue;
 			}
 
-			std::string tpath = getTextureCacheDir() + DIR_DELIM + name;
+			std::string sha1_decoded = base64_decode(sha1_texture);
-			// Read data
+			std::ostringstream tmp_os(std::ios_base::binary);
-			std::ifstream fis(tpath.c_str(), std::ios_base::binary);
+			bool tex_in_cache = m_texture_cache.loadByChecksum(name,
+					tmp_os, sha1_decoded);
+			m_texture_name_sha1_map.set(name, sha1_decoded);
 
+			if(tex_in_cache) {
 
-			if(fis.good() == false){
+				SHA1 sha1;
-				infostream<<"Client::Texture not found in cache: "
+				sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
-						<<name << " expected it at: "<<tpath<<std::endl;
-			}
-			else
-			{
-				std::ostringstream tmp_os(std::ios_base::binary);
-				bool bad = false;
-				for(;;){
-					char buf[1024];
-					fis.read(buf, 1024);
-					std::streamsize len = fis.gcount();
-					tmp_os.write(buf, len);
-					if(fis.eof())
-						break;
-					if(!fis.good()){
-						bad = true;
-						break;
-					}
-				}
-				if(bad){
-					infostream<<"Client: Failed to read texture from cache\""
-							<<name<<"\""<<std::endl;
-				}
-				else {
 
-					SHA1 sha1;
+				unsigned char *digest = sha1.getDigest();
-					sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
 
-					unsigned char *digest = sha1.getDigest();
+				std::string digest_string = base64_encode(digest, 20);
 
-					std::string digest_string = base64_encode(digest, 20);
+				if (digest_string == sha1_texture) {
+					// Silly irrlicht's const-incorrectness
+					Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
 
-					if (digest_string == sha1_texture) {
+					// Create an irrlicht memory file
-						// Silly irrlicht's const-incorrectness
+					io::IReadFile *rfile = irrfs->createMemoryReadFile(
-						Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
+							*data_rw,  tmp_os.str().size(), "_tempreadfile");
-
+					assert(rfile);
-						// Create an irrlicht memory file
+					// Read image
-						io::IReadFile *rfile = irrfs->createMemoryReadFile(
+					video::IImage *img = vdrv->createImageFromFile(rfile);
-								*data_rw,  tmp_os.str().size(), "_tempreadfile");
+					if(!img){
-						assert(rfile);
+						infostream<<"Client: Cannot create image from data of "
-						// Read image
+								<<"received texture \""<<name<<"\""<<std::endl;
-						video::IImage *img = vdrv->createImageFromFile(rfile);
+						rfile->drop();
-						if(!img){
-							infostream<<"Client: Cannot create image from data of "
-									<<"received texture \""<<name<<"\""<<std::endl;
-							rfile->drop();
-						}
-						else {
-							m_tsrc->insertSourceImage(name, img);
-							img->drop();
-							rfile->drop();
-
-							texture_found = true;
-						}
 					}
 					else {
-						infostream<<"Client::Texture cached sha1 hash not matching server hash: "
+						m_tsrc->insertSourceImage(name, img);
-								<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
+						img->drop();
-					}
+						rfile->drop();
 
-					free(digest);
+						texture_found = true;
+					}
 				}
+				else {
+					infostream<<"Client::Texture cached sha1 hash not matching server hash: "
+							<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
+				}
+
+				free(digest);
 			}
 
 			//add texture request
@@ -1598,15 +1576,15 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 
 			fs::CreateAllDirs(getTextureCacheDir());
 
-			std::string filename = getTextureCacheDir() + DIR_DELIM + name;
+			{
-			std::ofstream outfile(filename.c_str(), std::ios_base::binary | std::ios_base::trunc);
+				core::map<std::string, std::string>::Node *n;
-
+				n = m_texture_name_sha1_map.find(name);
-			if (outfile.good()) {
+				if(n == NULL)
-				outfile.write(data.c_str(),data.length());
+					errorstream<<"The server sent a texture that has not been announced."
-				outfile.close();
+						<<std::endl;
-			}
+				else
-			else {
+					m_texture_cache.updateByChecksum(name,
-				errorstream<<"Client: Unable to open cached texture file "<< filename <<std::endl;
+							data, n->getValue());
 			}
 
 			m_tsrc->insertSourceImage(name, img);
@@ -2382,6 +2360,10 @@ void Client::afterContentReceived()
 	assert(m_nodedef_received);
 	assert(m_textures_received);
 
+	// remove the information about which checksum each texture
+	// ought to have
+	m_texture_name_sha1_map.clear();
+
 	// Rebuild inherited images and recreate textures
 	m_tsrc->rebuildImagesAndTextures();
 

src/client.h

@@ -34,6 +34,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "gamedef.h"
 #include "inventorymanager.h"
 #include "filesys.h"
+#include "filecache.h"
 
 struct MeshMakeData;
 class MapBlockMesh;
@@ -366,6 +367,10 @@ private:
 	bool m_access_denied;
 	std::wstring m_access_denied_reason;
 	Queue<ClientEvent> m_client_event_queue;
+	FileCache m_texture_cache;
+	// a map of the name and SHA1 checksum of each texture;
+	// cleared after content has been recieved
+	core::map<std::string, std::string> m_texture_name_sha1_map;
 	float m_texture_receive_progress;
 	bool m_textures_received;
 	bool m_itemdef_received;

src/filecache.cpp

@@ -0,0 +1,117 @@
+/*
+Minetest-c55
+Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "filecache.h"
+#include "clientserver.h"
+#include "log.h"
+#include "filesys.h"
+#include "utility.h"
+#include "hex.h"
+
+#include <string>
+#include <iostream>
+
+bool FileCache::loadByPath(const std::string &name, std::ostream &os,
+		const std::string &path)
+{
+	std::ifstream fis(path.c_str(), std::ios_base::binary);
+
+	if(!fis.good()){
+		infostream<<"FileCache: File not found in cache: "
+			<<name << " expected it at: "<<path<<std::endl;
+		return false;
+	}
+
+	bool bad = false;
+	for(;;){
+		char buf[1024];
+		fis.read(buf, 1024);
+		std::streamsize len = fis.gcount();
+		os.write(buf, len);
+		if(fis.eof())
+			break;
+		if(!fis.good()){
+			bad = true;
+			break;
+		}
+	}
+	if(bad){
+		infostream<<"FileCache: Failed to read file from cache: \""
+			<<path<<"\""<<std::endl;
+	}
+
+	return !bad;
+}
+
+bool FileCache::updateByPath(const std::string &name, const std::string &data,
+		const std::string &path)
+{
+	std::ofstream file(path.c_str(), std::ios_base::binary |
+			std::ios_base::trunc);
+
+	if(!file.good())
+	{
+		errorstream<<"FileCache: Can't write to file at "
+			<<path<<std::endl;
+		return false;
+	}
+
+	file.write(data.c_str(), data.length());
+	file.close();
+
+	return !file.fail();
+}
+
+bool FileCache::loadByName(const std::string &name, std::ostream &os)
+{
+	std::string path = m_dir + DIR_DELIM + name;
+	return loadByPath(name, os, path);
+}
+
+
+bool FileCache::updateByName(const std::string &name, const std::string &data)
+{
+	std::string path = m_dir + DIR_DELIM + name;
+	return updateByPath(name, data, path);
+}
+
+std::string FileCache::getPathFromChecksum(const std::string &name,
+		const std::string &checksum)
+{
+	std::string checksum_hex = hex_encode(checksum.c_str(), checksum.length());
+	size_t dot = name.find_last_of('.');;
+	std::string ext = (dot == std::string::npos)? "" :
+		name.substr(dot, std::string::npos);
+	return m_dir + DIR_DELIM + checksum_hex + ext;
+}
+
+bool FileCache::loadByChecksum(const std::string &name, std::ostream &os,
+		const std::string &checksum)
+{
+	std::string path = getPathFromChecksum(name, checksum);
+	return loadByPath(name, os, path);
+}
+
+bool FileCache::updateByChecksum(const std::string &name,
+		const std::string &data, const std::string &checksum)
+{
+	std::string path = getPathFromChecksum(name, checksum);
+	return updateByPath(name, data, path);
+}

src/filecache.h

@@ -0,0 +1,79 @@
+/*
+Minetest-c55
+Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef FILECACHE_HEADER
+#define FILECACHE_HEADER
+
+#include <string>
+#include <iostream>
+
+class FileCache
+{
+public:
+	/*
+		'dir' is the file cache directory to use.
+	*/
+	FileCache(std::string dir):
+		m_dir(dir)
+	{
+	}
+
+	/*
+		Searches the cache for a file with a given name.
+		If the file is found, lookup copies it into 'os' and
+		returns true. Otherwise false is returned.
+	*/
+	bool loadByName(const std::string &name, std::ostream &os);
+
+	/*
+		Stores a file in the cache based on its name.
+		Returns true on success, false otherwise.
+	*/
+	bool updateByName(const std::string &name, const std::string &data);
+
+	/*
+		Loads a file based on a check sum, which may be any kind of
+		rather unique byte sequence. Returns true, if the file could
+		be written into os, false otherwise.
+		A file name is required to give the disk file a name that
+		has the right file name extension (e.g. ".png").
+	*/
+	bool loadByChecksum(const std::string &name, std::ostream &os,
+			const std::string &checksum);
+
+	/*
+		Stores a file in the cache based on its checksum.
+		Returns true on success, false otherwise.
+	*/
+	bool updateByChecksum(const std::string &name, const std::string &data,
+			const std::string &checksum);
+
+private:
+	std::string m_dir;
+
+	bool loadByPath(const std::string &name, std::ostream &os,
+			const std::string &path);
+	bool updateByPath(const std::string &name, const std::string &data,
+			const std::string &path);
+	std::string getPathFromChecksum(const std::string &name,
+			const std::string &checksum);
+};
+
+#endif

src/hex.h

@@ -0,0 +1,44 @@
+/*
+Minetest-c55
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef HEX_HEADER
+#define HEX_HEADER
+
+#include <string>
+
+static const char hex_chars[] = "0123456789abcdef";
+
+static std::string hex_encode(const char *data, unsigned int data_size)
+{
+	std::string ret;
+	char buf2[3];
+	buf2[2] = '\0';
+
+	for(unsigned int i = 0; i < data_size; i++)
+	{
+		unsigned char c = (unsigned char) data[i];
+		buf2[0] = hex_chars[(c & 0xf0) >> 4];
+		buf2[1] = hex_chars[c & 0x0f];
+		ret.append(buf2);
+	}
+
+	return ret;
+}
+
+#endif