Merge remote-tracking branch 'queatz/sqlite-map'

This commit is contained in:
Perttu Ahola 2011-09-06 18:04:02 +03:00
commit 310fa10035
5 changed files with 5738 additions and 2631 deletions

View File

@ -29,12 +29,21 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "mapgen.h"
#include "nodemetadata.h"
extern "C" {
#include "sqlite3.h"
}
/*
SQLite format specification:
- Initially only replaces sectors/ and sectors2/
If map.sqlite does not exist in the save dir
or the block was not found in the database
the map will try to load from sectors folder.
In either case, map.sqlite will be created
and all future saves will save there.
Structure of map.sqlite:
Tables:
blocks
(PK) INT pos
BLOB data
*/
/*
@ -1408,6 +1417,8 @@ void Map::timerUpdate(float dtime, float unload_timeout,
core::list<MapBlock*> blocks;
sector->getBlocks(blocks);
beginSave();
for(core::list<MapBlock*>::Iterator i = blocks.begin();
i != blocks.end(); i++)
{
@ -1440,6 +1451,7 @@ void Map::timerUpdate(float dtime, float unload_timeout,
all_blocks_deleted = false;
}
}
endSave();
if(all_blocks_deleted)
{
@ -1873,7 +1885,10 @@ void Map::nodeMetadataStep(float dtime,
ServerMap::ServerMap(std::string savedir):
Map(dout_server),
m_seed(0),
m_map_metadata_changed(true)
m_map_metadata_changed(true),
m_database(NULL),
m_database_read(NULL),
m_database_write(NULL)
{
dstream<<__FUNCTION_NAME<<std::endl;
@ -2001,6 +2016,16 @@ ServerMap::~ServerMap()
<<", exception: "<<e.what()<<std::endl;
}
/*
Close database if it was opened
*/
if(m_database_read)
sqlite3_finalize(m_database_read);
if(m_database_write)
sqlite3_finalize(m_database_write);
if(m_database)
sqlite3_close(m_database);
#if 0
/*
Free all MapChunks
@ -2314,6 +2339,7 @@ ServerMapSector * ServerMap::createSector(v2s16 p2d)
/*
Try to load metadata from disk
*/
#if 0
if(loadSectorMeta(p2d) == true)
{
ServerMapSector *sector = (ServerMapSector*)getSectorNoGenerateNoEx(p2d);
@ -2324,7 +2350,7 @@ ServerMapSector * ServerMap::createSector(v2s16 p2d)
}
return sector;
}
#endif
/*
Do not create over-limit
*/
@ -2766,6 +2792,75 @@ plan_b:
//return (s16)level;
}
void ServerMap::createDatabase() {
int e;
assert(m_database);
e = sqlite3_exec(m_database,
"CREATE TABLE IF NOT EXISTS `blocks` ("
"`pos` INT NOT NULL PRIMARY KEY,"
"`data` BLOB"
");"
, NULL, NULL, NULL);
if(e == SQLITE_ABORT)
throw FileNotGoodException("Could not create database structure");
else
dstream<<"Server: Database structure was created";
}
void ServerMap::verifyDatabase() {
if(m_database)
return;
{
std::string dbp = m_savedir + "/map.sqlite";
bool needs_create = false;
int d;
/*
Open the database connection
*/
createDirs(m_savedir);
if(!fs::PathExists(dbp))
needs_create = true;
d = sqlite3_open_v2(dbp.c_str(), &m_database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
if(d != SQLITE_OK) {
dstream<<"WARNING: Database failed to open: "<<sqlite3_errmsg(m_database)<<std::endl;
throw FileNotGoodException("Cannot open database file");
}
if(needs_create)
createDatabase();
d = sqlite3_prepare(m_database, "SELECT `data` FROM `blocks` WHERE `pos`=? LIMIT 1", -1, &m_database_read, NULL);
if(d != SQLITE_OK) {
dstream<<"WARNING: Database read statment failed to prepare: "<<sqlite3_errmsg(m_database)<<std::endl;
throw FileNotGoodException("Cannot prepare read statement");
}
d = sqlite3_prepare(m_database, "REPLACE INTO `blocks` VALUES(?, ?)", -1, &m_database_write, NULL);
if(d != SQLITE_OK) {
dstream<<"WARNING: Database write statment failed to prepare: "<<sqlite3_errmsg(m_database)<<std::endl;
throw FileNotGoodException("Cannot prepare write statement");
}
dstream<<"Server: Database opened"<<std::endl;
}
}
bool ServerMap::loadFromFolders() {
if(!m_database && !fs::PathExists(m_savedir + "/map.sqlite"))
return true;
return false;
}
sqlite3_int64 ServerMap::getBlockAsInteger(const v3s16 pos) {
return (sqlite3_int64)pos.Z*16777216 +
(sqlite3_int64)pos.Y*4096 + (sqlite3_int64)pos.X;
}
void ServerMap::createDirs(std::string path)
{
if(fs::CreateAllDirs(path) == false)
@ -2869,6 +2964,7 @@ void ServerMap::save(bool only_changed)
u32 block_count = 0;
u32 block_count_all = 0; // Number of blocks in memory
beginSave();
core::map<v2s16, MapSector*>::Iterator i = m_sectors.getIterator();
for(; i.atEnd() == false; i++)
{
@ -2883,6 +2979,8 @@ void ServerMap::save(bool only_changed)
core::list<MapBlock*> blocks;
sector->getBlocks(blocks);
core::list<MapBlock*>::Iterator j;
//sqlite3_exec(m_database, "BEGIN;", NULL, NULL, NULL);
for(j=blocks.begin(); j!=blocks.end(); j++)
{
MapBlock *block = *j;
@ -2901,8 +2999,10 @@ void ServerMap::save(bool only_changed)
<<block->getPos().Z<<")"
<<std::endl;*/
}
//sqlite3_exec(m_database, "COMMIT;", NULL, NULL, NULL);
}
}
endSave();
/*
Only print if something happened or saved whole map
@ -3161,6 +3261,18 @@ bool ServerMap::loadSectorFull(v2s16 p2d)
}
#endif
void ServerMap::beginSave() {
verifyDatabase();
if(sqlite3_exec(m_database, "BEGIN;", NULL, NULL, NULL) != SQLITE_OK)
dstream<<"WARNING: beginSave() failed, saving might be slow.";
}
void ServerMap::endSave() {
verifyDatabase();
if(sqlite3_exec(m_database, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK)
dstream<<"WARNING: endSave() failed, map might not have saved.";
}
void ServerMap::saveBlock(MapBlock *block)
{
DSTACK(__FUNCTION_NAME);
@ -3180,6 +3292,8 @@ void ServerMap::saveBlock(MapBlock *block)
// Get destination
v3s16 p3d = block->getPos();
#if 0
v2s16 p2d(p3d.X, p3d.Z);
std::string sectordir = getSectorDir(p2d);
@ -3189,11 +3303,16 @@ void ServerMap::saveBlock(MapBlock *block)
std::ofstream o(fullpath.c_str(), std::ios_base::binary);
if(o.good() == false)
throw FileNotGoodException("Cannot open block data");
#endif
/*
[0] u8 serialization version
[1] data
*/
verifyDatabase();
std::ostringstream o(std::ios_base::binary);
o.write((char*)&version, 1);
// Write basic data
@ -3201,7 +3320,23 @@ void ServerMap::saveBlock(MapBlock *block)
// Write extra data stored on disk
block->serializeDiskExtra(o, version);
// Write block to database
std::string tmp = o.str();
const char *bytes = tmp.c_str();
if(sqlite3_bind_int64(m_database_write, 1, getBlockAsInteger(p3d)) != SQLITE_OK)
dstream<<"WARNING: Block position failed to bind: "<<sqlite3_errmsg(m_database)<<std::endl;
if(sqlite3_bind_blob(m_database_write, 2, (void *)bytes, o.tellp(), NULL) != SQLITE_OK) // TODO this mught not be the right length
dstream<<"WARNING: Block data failed to bind: "<<sqlite3_errmsg(m_database)<<std::endl;
int written = sqlite3_step(m_database_write);
if(written != SQLITE_DONE)
dstream<<"WARNING: Block failed to save ("<<p3d.X<<", "<<p3d.Y<<", "<<p3d.Z<<") "
<<sqlite3_errmsg(m_database)<<std::endl;
// Make ready for later reuse
sqlite3_reset(m_database_write);
// We just wrote it to the disk so clear modified flag
block->resetModified();
}
@ -3282,12 +3417,111 @@ void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSecto
}
}
void ServerMap::loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load)
{
DSTACK(__FUNCTION_NAME);
try {
std::istringstream is(*blob, std::ios_base::binary);
u8 version = SER_FMT_VER_INVALID;
is.read((char*)&version, 1);
if(is.fail())
throw SerializationError("ServerMap::loadBlock(): Failed"
" to read MapBlock version");
/*u32 block_size = MapBlock::serializedLength(version);
SharedBuffer<u8> data(block_size);
is.read((char*)*data, block_size);*/
// This will always return a sector because we're the server
//MapSector *sector = emergeSector(p2d);
MapBlock *block = NULL;
bool created_new = false;
block = sector->getBlockNoCreateNoEx(p3d.Y);
if(block == NULL)
{
block = sector->createBlankBlockNoInsert(p3d.Y);
created_new = true;
}
// Read basic data
block->deSerialize(is, version);
// Read extra data stored on disk
block->deSerializeDiskExtra(is, version);
// If it's a new block, insert it to the map
if(created_new)
sector->insertBlock(block);
/*
Save blocks loaded in old format in new format
*/
if(version < SER_FMT_VER_HIGHEST || save_after_load)
{
saveBlock(block);
}
// We just loaded it from, so it's up-to-date.
block->resetModified();
}
catch(SerializationError &e)
{
dstream<<"WARNING: Invalid block data in database "
<<" (SerializationError). "
<<"what()="<<e.what()
<<std::endl;
//" Ignoring. A new one will be generated.
assert(0);
// TODO: Copy to a backup database.
}
}
MapBlock* ServerMap::loadBlock(v3s16 blockpos)
{
DSTACK(__FUNCTION_NAME);
v2s16 p2d(blockpos.X, blockpos.Z);
if(!loadFromFolders()) {
verifyDatabase();
if(sqlite3_bind_int64(m_database_read, 1, getBlockAsInteger(blockpos)) != SQLITE_OK)
dstream<<"WARNING: Could not bind block position for load: "
<<sqlite3_errmsg(m_database)<<std::endl;
if(sqlite3_step(m_database_read) == SQLITE_ROW) {
/*
Make sure sector is loaded
*/
MapSector *sector = createSector(p2d);
/*
Load block
*/
const char * data = (const char *)sqlite3_column_blob(m_database_read, 0);
size_t len = sqlite3_column_bytes(m_database_read, 0);
std::string datastr(data, len);
loadBlock(&datastr, blockpos, sector, false);
sqlite3_step(m_database_read);
// We should never get more than 1 row, so ok to reset
sqlite3_reset(m_database_read);
return getBlockNoCreateNoEx(blockpos);
}
sqlite3_reset(m_database_read);
// Not found in database, try the files
}
// The directory layout we're going to load from.
// 1 - original sectors/xxxxzzzz/
// 2 - new sectors2/xxx/zzz/
@ -3338,9 +3572,9 @@ MapBlock* ServerMap::loadBlock(v3s16 blockpos)
return NULL;
/*
Load block
Load block and save it to the database
*/
loadBlock(sectordir, blockfilename, sector, loadlayout != 2);
loadBlock(sectordir, blockfilename, sector, true);
return getBlockNoCreateNoEx(blockpos);
}

View File

@ -24,6 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include <jmutexautolock.h>
#include <jthread.h>
#include <iostream>
#include <sstream>
#include "common_irrlicht.h"
#include "mapnode.h"
@ -31,6 +32,10 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "constants.h"
#include "voxel.h"
extern "C" {
#include "sqlite3.h"
}
class MapSector;
class ServerMapSector;
class ClientMapSector;
@ -220,6 +225,10 @@ public:
//core::aabbox3d<s16> getDisplayedBlockArea();
//bool updateChangedVisibleArea();
// Call these before and after saving of many blocks
virtual void beginSave() {return;};
virtual void endSave() {return;};
virtual void save(bool only_changed){assert(0);};
@ -361,6 +370,23 @@ public:
v3s16 getBlockPos(std::string sectordir, std::string blockfile);
static std::string getBlockFilename(v3s16 p);
/*
Database functions
*/
// Create the database structure
void createDatabase();
// Verify we can read/write to the database
void verifyDatabase();
// Get an integer suitable for a block
static sqlite3_int64 getBlockAsInteger(const v3s16 pos);
// Returns true if the database file does not exist
bool loadFromFolders();
// Call these before and after saving of blocks
void beginSave();
void endSave();
void save(bool only_changed);
//void loadAll();
@ -391,6 +417,8 @@ public:
// This will generate a sector with getSector if not found.
void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load=false);
MapBlock* loadBlock(v3s16 p);
// Database version
void loadBlock(std::string *blob, v3s16 p3d, MapSector *sector, bool save_after_load=false);
// For debug printing
virtual void PrintInfo(std::ostream &out);
@ -419,6 +447,13 @@ private:
This is reset to false when written on disk.
*/
bool m_map_metadata_changed;
/*
SQLite database and statements
*/
sqlite3 *m_database;
sqlite3_stmt *m_database_read;
sqlite3_stmt *m_database_write;
};
/*

File diff suppressed because it is too large Load Diff

View File

@ -107,9 +107,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
#define SQLITE_VERSION "3.7.6.3"
#define SQLITE_VERSION_NUMBER 3007006
#define SQLITE_SOURCE_ID "2011-05-19 13:26:54 ed1da510a239ea767a01dc332b667119fa3c908e"
#define SQLITE_VERSION "3.7.7.1"
#define SQLITE_VERSION_NUMBER 3007007
#define SQLITE_SOURCE_ID "2011-06-28 17:39:05 af0d91adf497f5f36ec3813f04235a6e195a605f"
/*
** CAPI3REF: Run-Time Library Version Numbers
@ -310,7 +310,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
** argument. ^If the callback function of the 3rd argument to
** sqlite3_exec() is not NULL, then it is invoked for each result row
** coming out of the evaluated SQL statements. ^The 4th argument to
** to sqlite3_exec() is relayed through to the 1st argument of each
** sqlite3_exec() is relayed through to the 1st argument of each
** callback invocation. ^If the callback pointer to sqlite3_exec()
** is NULL, then no callback is ever invoked and result rows are
** ignored.
@ -375,7 +375,8 @@ SQLITE_API int sqlite3_exec(
**
** New error codes may be added in future versions of SQLite.
**
** See also: [SQLITE_IOERR_READ | extended result codes]
** See also: [SQLITE_IOERR_READ | extended result codes],
** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes].
*/
#define SQLITE_OK 0 /* Successful result */
/* beginning-of-error-codes */
@ -452,17 +453,21 @@ SQLITE_API int sqlite3_exec(
#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8))
#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8))
#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8))
#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8))
#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8))
#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8))
#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8))
#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8))
#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8))
#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8))
#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8))
/*
** CAPI3REF: Flags For File Open Operations
**
** These bit values are intended for use in the
** 3rd parameter to the [sqlite3_open_v2()] interface and
** in the 4th parameter to the xOpen method of the
** [sqlite3_vfs] object.
** in the 4th parameter to the [sqlite3_vfs.xOpen] method.
*/
#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */
#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */
@ -470,6 +475,7 @@ SQLITE_API int sqlite3_exec(
#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */
#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */
#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */
#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */
#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */
#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */
#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */
@ -580,17 +586,18 @@ struct sqlite3_file {
/*
** CAPI3REF: OS Interface File Virtual Methods Object
**
** Every file opened by the [sqlite3_vfs] xOpen method populates an
** Every file opened by the [sqlite3_vfs.xOpen] method populates an
** [sqlite3_file] object (or, more commonly, a subclass of the
** [sqlite3_file] object) with a pointer to an instance of this object.
** This object defines the methods used to perform various operations
** against the open file represented by the [sqlite3_file] object.
**
** If the xOpen method sets the sqlite3_file.pMethods element
** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element
** to a non-NULL pointer, then the sqlite3_io_methods.xClose method
** may be invoked even if the xOpen reported that it failed. The
** only way to prevent a call to xClose following a failed xOpen
** is for the xOpen to set the sqlite3_file.pMethods element to NULL.
** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The
** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen]
** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element
** to NULL.
**
** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or
** [SQLITE_SYNC_FULL]. The first choice is the normal fsync().
@ -762,7 +769,8 @@ typedef struct sqlite3_mutex sqlite3_mutex;
**
** An instance of the sqlite3_vfs object defines the interface between
** the SQLite core and the underlying operating system. The "vfs"
** in the name of the object stands for "virtual file system".
** in the name of the object stands for "virtual file system". See
** the [VFS | VFS documentation] for further information.
**
** The value of the iVersion field is initially 1 but may be larger in
** future versions of SQLite. Additional fields may be appended to this
@ -791,6 +799,7 @@ typedef struct sqlite3_mutex sqlite3_mutex;
** The zName field holds the name of the VFS module. The name must
** be unique across all VFS modules.
**
** [[sqlite3_vfs.xOpen]]
** ^SQLite guarantees that the zFilename parameter to xOpen
** is either a NULL pointer or string obtained
** from xFullPathname() with an optional suffix added.
@ -868,6 +877,7 @@ typedef struct sqlite3_mutex sqlite3_mutex;
** element will be valid after xOpen returns regardless of the success
** or failure of the xOpen call.
**
** [[sqlite3_vfs.xAccess]]
** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS]
** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to
** test whether a file is readable and writable, or [SQLITE_ACCESS_READ]
@ -892,7 +902,7 @@ typedef struct sqlite3_mutex sqlite3_mutex;
** method returns a Julian Day Number for the current date and time as
** a floating point value.
** ^The xCurrentTimeInt64() method returns, as an integer, the Julian
** Day Number multipled by 86400000 (the number of milliseconds in
** Day Number multiplied by 86400000 (the number of milliseconds in
** a 24-hour day).
** ^SQLite will use the xCurrentTimeInt64() method to get the current
** date and time if that method is available (if iVersion is 2 or
@ -1114,9 +1124,9 @@ SQLITE_API int sqlite3_os_end(void);
** implementation of an application-defined [sqlite3_os_init()].
**
** The first argument to sqlite3_config() is an integer
** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines
** [configuration option] that determines
** what property of SQLite is to be configured. Subsequent arguments
** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option]
** vary depending on the [configuration option]
** in the first argument.
**
** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK].
@ -1226,6 +1236,7 @@ struct sqlite3_mem_methods {
/*
** CAPI3REF: Configuration Options
** KEYWORDS: {configuration option}
**
** These constants are the available integer configuration options that
** can be passed as the first argument to the [sqlite3_config()] interface.
@ -1238,7 +1249,7 @@ struct sqlite3_mem_methods {
** is invoked.
**
** <dl>
** <dt>SQLITE_CONFIG_SINGLETHREAD</dt>
** [[SQLITE_CONFIG_SINGLETHREAD]] <dt>SQLITE_CONFIG_SINGLETHREAD</dt>
** <dd>There are no arguments to this option. ^This option sets the
** [threading mode] to Single-thread. In other words, it disables
** all mutexing and puts SQLite into a mode where it can only be used
@ -1249,7 +1260,7 @@ struct sqlite3_mem_methods {
** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD
** configuration option.</dd>
**
** <dt>SQLITE_CONFIG_MULTITHREAD</dt>
** [[SQLITE_CONFIG_MULTITHREAD]] <dt>SQLITE_CONFIG_MULTITHREAD</dt>
** <dd>There are no arguments to this option. ^This option sets the
** [threading mode] to Multi-thread. In other words, it disables
** mutexing on [database connection] and [prepared statement] objects.
@ -1263,7 +1274,7 @@ struct sqlite3_mem_methods {
** [sqlite3_config()] will return [SQLITE_ERROR] if called with the
** SQLITE_CONFIG_MULTITHREAD configuration option.</dd>
**
** <dt>SQLITE_CONFIG_SERIALIZED</dt>
** [[SQLITE_CONFIG_SERIALIZED]] <dt>SQLITE_CONFIG_SERIALIZED</dt>
** <dd>There are no arguments to this option. ^This option sets the
** [threading mode] to Serialized. In other words, this option enables
** all mutexes including the recursive
@ -1279,7 +1290,7 @@ struct sqlite3_mem_methods {
** [sqlite3_config()] will return [SQLITE_ERROR] if called with the
** SQLITE_CONFIG_SERIALIZED configuration option.</dd>
**
** <dt>SQLITE_CONFIG_MALLOC</dt>
** [[SQLITE_CONFIG_MALLOC]] <dt>SQLITE_CONFIG_MALLOC</dt>
** <dd> ^(This option takes a single argument which is a pointer to an
** instance of the [sqlite3_mem_methods] structure. The argument specifies
** alternative low-level memory allocation routines to be used in place of
@ -1287,7 +1298,7 @@ struct sqlite3_mem_methods {
** its own private copy of the content of the [sqlite3_mem_methods] structure
** before the [sqlite3_config()] call returns.</dd>
**
** <dt>SQLITE_CONFIG_GETMALLOC</dt>
** [[SQLITE_CONFIG_GETMALLOC]] <dt>SQLITE_CONFIG_GETMALLOC</dt>
** <dd> ^(This option takes a single argument which is a pointer to an
** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods]
** structure is filled with the currently defined memory allocation routines.)^
@ -1295,7 +1306,7 @@ struct sqlite3_mem_methods {
** routines with a wrapper that simulations memory allocation failure or
** tracks memory usage, for example. </dd>
**
** <dt>SQLITE_CONFIG_MEMSTATUS</dt>
** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt>
** <dd> ^This option takes single argument of type int, interpreted as a
** boolean, which enables or disables the collection of memory allocation
** statistics. ^(When memory allocation statistics are disabled, the
@ -1311,7 +1322,7 @@ struct sqlite3_mem_methods {
** allocation statistics are disabled by default.
** </dd>
**
** <dt>SQLITE_CONFIG_SCRATCH</dt>
** [[SQLITE_CONFIG_SCRATCH]] <dt>SQLITE_CONFIG_SCRATCH</dt>
** <dd> ^This option specifies a static memory buffer that SQLite can use for
** scratch memory. There are three arguments: A pointer an 8-byte
** aligned memory buffer from which the scratch allocations will be
@ -1327,9 +1338,9 @@ struct sqlite3_mem_methods {
** scratch memory beyond what is provided by this configuration option, then
** [sqlite3_malloc()] will be used to obtain the memory needed.</dd>
**
** <dt>SQLITE_CONFIG_PAGECACHE</dt>
** [[SQLITE_CONFIG_PAGECACHE]] <dt>SQLITE_CONFIG_PAGECACHE</dt>
** <dd> ^This option specifies a static memory buffer that SQLite can use for
** the database page cache with the default page cache implemenation.
** the database page cache with the default page cache implementation.
** This configuration should not be used if an application-define page
** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option.
** There are three arguments to this option: A pointer to 8-byte aligned
@ -1348,7 +1359,7 @@ struct sqlite3_mem_methods {
** be aligned to an 8-byte boundary or subsequent behavior of SQLite
** will be undefined.</dd>
**
** <dt>SQLITE_CONFIG_HEAP</dt>
** [[SQLITE_CONFIG_HEAP]] <dt>SQLITE_CONFIG_HEAP</dt>
** <dd> ^This option specifies a static memory buffer that SQLite will use
** for all of its dynamic memory allocation needs beyond those provided
** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE].
@ -1365,7 +1376,7 @@ struct sqlite3_mem_methods {
** The minimum allocation size is capped at 2^12. Reasonable values
** for the minimum allocation size are 2^5 through 2^8.</dd>
**
** <dt>SQLITE_CONFIG_MUTEX</dt>
** [[SQLITE_CONFIG_MUTEX]] <dt>SQLITE_CONFIG_MUTEX</dt>
** <dd> ^(This option takes a single argument which is a pointer to an
** instance of the [sqlite3_mutex_methods] structure. The argument specifies
** alternative low-level mutex routines to be used in place
@ -1377,7 +1388,7 @@ struct sqlite3_mem_methods {
** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will
** return [SQLITE_ERROR].</dd>
**
** <dt>SQLITE_CONFIG_GETMUTEX</dt>
** [[SQLITE_CONFIG_GETMUTEX]] <dt>SQLITE_CONFIG_GETMUTEX</dt>
** <dd> ^(This option takes a single argument which is a pointer to an
** instance of the [sqlite3_mutex_methods] structure. The
** [sqlite3_mutex_methods]
@ -1390,7 +1401,7 @@ struct sqlite3_mem_methods {
** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will
** return [SQLITE_ERROR].</dd>
**
** <dt>SQLITE_CONFIG_LOOKASIDE</dt>
** [[SQLITE_CONFIG_LOOKASIDE]] <dt>SQLITE_CONFIG_LOOKASIDE</dt>
** <dd> ^(This option takes two arguments that determine the default
** memory allocation for the lookaside memory allocator on each
** [database connection]. The first argument is the
@ -1400,18 +1411,18 @@ struct sqlite3_mem_methods {
** verb to [sqlite3_db_config()] can be used to change the lookaside
** configuration on individual connections.)^ </dd>
**
** <dt>SQLITE_CONFIG_PCACHE</dt>
** [[SQLITE_CONFIG_PCACHE]] <dt>SQLITE_CONFIG_PCACHE</dt>
** <dd> ^(This option takes a single argument which is a pointer to
** an [sqlite3_pcache_methods] object. This object specifies the interface
** to a custom page cache implementation.)^ ^SQLite makes a copy of the
** object and uses it for page cache memory allocations.</dd>
**
** <dt>SQLITE_CONFIG_GETPCACHE</dt>
** [[SQLITE_CONFIG_GETPCACHE]] <dt>SQLITE_CONFIG_GETPCACHE</dt>
** <dd> ^(This option takes a single argument which is a pointer to an
** [sqlite3_pcache_methods] object. SQLite copies of the current
** page cache implementation into that object.)^ </dd>
**
** <dt>SQLITE_CONFIG_LOG</dt>
** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt>
** <dd> ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a
** function with a call signature of void(*)(void*,int,const char*),
** and a pointer to void. ^If the function pointer is not NULL, it is
@ -1429,6 +1440,18 @@ struct sqlite3_mem_methods {
** In a multi-threaded application, the application-defined logger
** function must be threadsafe. </dd>
**
** [[SQLITE_CONFIG_URI]] <dt>SQLITE_CONFIG_URI
** <dd> This option takes a single argument of type int. If non-zero, then
** URI handling is globally enabled. If the parameter is zero, then URI handling
** is globally disabled. If URI handling is globally enabled, all filenames
** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or
** specified as part of [ATTACH] commands are interpreted as URIs, regardless
** of whether or not the [SQLITE_OPEN_URI] flag is set when the database
** connection is opened. If it is globally disabled, filenames are
** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the
** database connection is opened. By default, URI handling is globally
** disabled. The default value may be changed by compiling with the
** [SQLITE_USE_URI] symbol defined.
** </dl>
*/
#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */
@ -1447,6 +1470,7 @@ struct sqlite3_mem_methods {
#define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */
#define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */
#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */
#define SQLITE_CONFIG_URI 17 /* int */
/*
** CAPI3REF: Database Connection Configuration Options
@ -1532,13 +1556,17 @@ SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff);
**
** ^This routine returns the [rowid] of the most recent
** successful [INSERT] into the database from the [database connection]
** in the first argument. ^If no successful [INSERT]s
** in the first argument. ^As of SQLite version 3.7.7, this routines
** records the last insert rowid of both ordinary tables and [virtual tables].
** ^If no successful [INSERT]s
** have ever occurred on that database connection, zero is returned.
**
** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted
** row is returned by this routine as long as the trigger is running.
** But once the trigger terminates, the value returned by this routine
** reverts to the last value inserted before the trigger fired.)^
** ^(If an [INSERT] occurs within a trigger or within a [virtual table]
** method, then this routine will return the [rowid] of the inserted
** row as long as the trigger or virtual table method is running.
** But once the trigger or virtual table method ends, the value returned
** by this routine reverts to what it was before the trigger or virtual
** table method began.)^
**
** ^An [INSERT] that fails due to a constraint violation is not a
** successful [INSERT] and does not change the value returned by this
@ -2201,6 +2229,9 @@ SQLITE_API int sqlite3_set_authorizer(
** to signal SQLite whether or not the action is permitted. See the
** [sqlite3_set_authorizer | authorizer documentation] for additional
** information.
**
** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code]
** from the [sqlite3_vtab_on_conflict()] interface.
*/
#define SQLITE_DENY 1 /* Abort the SQL statement with an error */
#define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */
@ -2323,7 +2354,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
/*
** CAPI3REF: Opening A New Database Connection
**
** ^These routines open an SQLite database file whose name is given by the
** ^These routines open an SQLite database file as specified by the
** filename argument. ^The filename argument is interpreted as UTF-8 for
** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte
** order for sqlite3_open16(). ^(A [database connection] handle is usually
@ -2350,7 +2381,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** sqlite3_open_v2() can take one of
** the following three values, optionally combined with the
** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE],
** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^
** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^
**
** <dl>
** ^(<dt>[SQLITE_OPEN_READONLY]</dt>
@ -2369,9 +2400,8 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** </dl>
**
** If the 3rd parameter to sqlite3_open_v2() is not one of the
** combinations shown above or one of the combinations shown above combined
** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX],
** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags,
** combinations shown above optionally combined with other
** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits]
** then the behavior is undefined.
**
** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection
@ -2386,6 +2416,11 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not
** participate in [shared cache mode] even if it is enabled.
**
** ^The fourth parameter to sqlite3_open_v2() is the name of the
** [sqlite3_vfs] object that defines the operating system interface that
** the new database connection should use. ^If the fourth parameter is
** a NULL pointer then the default [sqlite3_vfs] object is used.
**
** ^If the filename is ":memory:", then a private, temporary in-memory database
** is created for the connection. ^This in-memory database will vanish when
** the database connection is closed. Future versions of SQLite might
@ -2398,10 +2433,111 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
** on-disk database will be created. ^This private database will be
** automatically deleted as soon as the database connection is closed.
**
** ^The fourth parameter to sqlite3_open_v2() is the name of the
** [sqlite3_vfs] object that defines the operating system interface that
** the new database connection should use. ^If the fourth parameter is
** a NULL pointer then the default [sqlite3_vfs] object is used.
** [[URI filenames in sqlite3_open()]] <h3>URI Filenames</h3>
**
** ^If [URI filename] interpretation is enabled, and the filename argument
** begins with "file:", then the filename is interpreted as a URI. ^URI
** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is
** set in the fourth argument to sqlite3_open_v2(), or if it has
** been enabled globally using the [SQLITE_CONFIG_URI] option with the
** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option.
** As of SQLite version 3.7.7, URI filename interpretation is turned off
** by default, but future releases of SQLite might enable URI filename
** interpretation by default. See "[URI filenames]" for additional
** information.
**
** URI filenames are parsed according to RFC 3986. ^If the URI contains an
** authority, then it must be either an empty string or the string
** "localhost". ^If the authority is not an empty string or "localhost", an
** error is returned to the caller. ^The fragment component of a URI, if
** present, is ignored.
**
** ^SQLite uses the path component of the URI as the name of the disk file
** which contains the database. ^If the path begins with a '/' character,
** then it is interpreted as an absolute path. ^If the path does not begin
** with a '/' (meaning that the authority section is omitted from the URI)
** then the path is interpreted as a relative path.
** ^On windows, the first component of an absolute path
** is a drive specification (e.g. "C:").
**
** [[core URI query parameters]]
** The query component of a URI may contain parameters that are interpreted
** either by SQLite itself, or by a [VFS | custom VFS implementation].
** SQLite interprets the following three query parameters:
**
** <ul>
** <li> <b>vfs</b>: ^The "vfs" parameter may be used to specify the name of
** a VFS object that provides the operating system interface that should
** be used to access the database file on disk. ^If this option is set to
** an empty string the default VFS object is used. ^Specifying an unknown
** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is
** present, then the VFS specified by the option takes precedence over
** the value passed as the fourth parameter to sqlite3_open_v2().
**
** <li> <b>mode</b>: ^(The mode parameter may be set to either "ro", "rw" or
** "rwc". Attempting to set it to any other value is an error)^.
** ^If "ro" is specified, then the database is opened for read-only
** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the
** third argument to sqlite3_prepare_v2(). ^If the mode option is set to
** "rw", then the database is opened for read-write (but not create)
** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had
** been set. ^Value "rwc" is equivalent to setting both
** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If sqlite3_open_v2() is
** used, it is an error to specify a value for the mode parameter that is
** less restrictive than that specified by the flags passed as the third
** parameter.
**
** <li> <b>cache</b>: ^The cache parameter may be set to either "shared" or
** "private". ^Setting it to "shared" is equivalent to setting the
** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to
** sqlite3_open_v2(). ^Setting the cache parameter to "private" is
** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit.
** ^If sqlite3_open_v2() is used and the "cache" parameter is present in
** a URI filename, its value overrides any behaviour requested by setting
** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag.
** </ul>
**
** ^Specifying an unknown parameter in the query component of a URI is not an
** error. Future versions of SQLite might understand additional query
** parameters. See "[query parameters with special meaning to SQLite]" for
** additional information.
**
** [[URI filename examples]] <h3>URI filename examples</h3>
**
** <table border="1" align=center cellpadding=5>
** <tr><th> URI filenames <th> Results
** <tr><td> file:data.db <td>
** Open the file "data.db" in the current directory.
** <tr><td> file:/home/fred/data.db<br>
** file:///home/fred/data.db <br>
** file://localhost/home/fred/data.db <br> <td>
** Open the database file "/home/fred/data.db".
** <tr><td> file://darkstar/home/fred/data.db <td>
** An error. "darkstar" is not a recognized authority.
** <tr><td style="white-space:nowrap">
** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db
** <td> Windows only: Open the file "data.db" on fred's desktop on drive
** C:. Note that the %20 escaping in this example is not strictly
** necessary - space characters can be used literally
** in URI filenames.
** <tr><td> file:data.db?mode=ro&cache=private <td>
** Open file "data.db" in the current directory for read-only access.
** Regardless of whether or not shared-cache mode is enabled by
** default, use a private cache.
** <tr><td> file:/home/fred/data.db?vfs=unix-nolock <td>
** Open file "/home/fred/data.db". Use the special VFS "unix-nolock".
** <tr><td> file:data.db?mode=readonly <td>
** An error. "readonly" is not a valid option for the "mode" parameter.
** </table>
**
** ^URI hexadecimal escape sequences (%HH) are supported within the path and
** query components of a URI. A hexadecimal escape sequence consists of a
** percent sign - "%" - followed by exactly two hexadecimal digits
** specifying an octet value. ^Before the path or query components of a
** URI filename are interpreted, they are encoded using UTF-8 and all
** hexadecimal escape sequences replaced by a single byte containing the
** corresponding octet. If this process generates an invalid UTF-8 encoding,
** the results are undefined.
**
** <b>Note to Windows users:</b> The encoding used for the filename argument
** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever
@ -2424,6 +2560,26 @@ SQLITE_API int sqlite3_open_v2(
const char *zVfs /* Name of VFS module to use */
);
/*
** CAPI3REF: Obtain Values For URI Parameters
**
** This is a utility routine, useful to VFS implementations, that checks
** to see if a database file was a URI that contained a specific query
** parameter, and if so obtains the value of the query parameter.
**
** The zFilename argument is the filename pointer passed into the xOpen()
** method of a VFS implementation. The zParam argument is the name of the
** query parameter we seek. This routine returns the value of the zParam
** parameter if it exists. If the parameter does not exist, this routine
** returns a NULL pointer.
**
** If the zFilename argument to this function is not a pointer that SQLite
** passed into the xOpen VFS method, then the behavior of this routine
** is undefined and probably undesirable.
*/
SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam);
/*
** CAPI3REF: Error Codes And Messages
**
@ -2539,43 +2695,45 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
** Additional information is available at [limits | Limits in SQLite].
**
** <dl>
** ^(<dt>SQLITE_LIMIT_LENGTH</dt>
** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt>
** <dd>The maximum size of any string or BLOB or table row, in bytes.<dd>)^
**
** ^(<dt>SQLITE_LIMIT_SQL_LENGTH</dt>
** [[SQLITE_LIMIT_SQL_LENGTH]] ^(<dt>SQLITE_LIMIT_SQL_LENGTH</dt>
** <dd>The maximum length of an SQL statement, in bytes.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_COLUMN</dt>
** [[SQLITE_LIMIT_COLUMN]] ^(<dt>SQLITE_LIMIT_COLUMN</dt>
** <dd>The maximum number of columns in a table definition or in the
** result set of a [SELECT] or the maximum number of columns in an index
** or in an ORDER BY or GROUP BY clause.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt>
** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt>
** <dd>The maximum depth of the parse tree on any expression.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt>
** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt>
** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_VDBE_OP</dt>
** [[SQLITE_LIMIT_VDBE_OP]] ^(<dt>SQLITE_LIMIT_VDBE_OP</dt>
** <dd>The maximum number of instructions in a virtual machine program
** used to implement an SQL statement. This limit is not currently
** enforced, though that might be added in some future release of
** SQLite.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_FUNCTION_ARG</dt>
** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(<dt>SQLITE_LIMIT_FUNCTION_ARG</dt>
** <dd>The maximum number of arguments on a function.</dd>)^
**
** ^(<dt>SQLITE_LIMIT_ATTACHED</dt>
** [[SQLITE_LIMIT_ATTACHED]] ^(<dt>SQLITE_LIMIT_ATTACHED</dt>
** <dd>The maximum number of [ATTACH | attached databases].)^</dd>
**
** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]]
** ^(<dt>SQLITE_LIMIT_LIKE_PATTERN_LENGTH</dt>
** <dd>The maximum length of the pattern argument to the [LIKE] or
** [GLOB] operators.</dd>)^
**
** [[SQLITE_LIMIT_VARIABLE_NUMBER]]
** ^(<dt>SQLITE_LIMIT_VARIABLE_NUMBER</dt>
** <dd>The maximum index number of any [parameter] in an SQL statement.)^
**
** ^(<dt>SQLITE_LIMIT_TRIGGER_DEPTH</dt>
** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(<dt>SQLITE_LIMIT_TRIGGER_DEPTH</dt>
** <dd>The maximum depth of recursion for triggers.</dd>)^
** </dl>
*/
@ -3104,7 +3262,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
** ^[SQLITE_BUSY] means that the database engine was unable to acquire the
** database locks it needs to do its job. ^If the statement is a [COMMIT]
** or occurs outside of an explicit transaction, then you can retry the
** statement. If the statement is not a [COMMIT] and occurs within a
** statement. If the statement is not a [COMMIT] and occurs within an
** explicit transaction then you should rollback the transaction before
** continuing.
**
@ -3383,7 +3541,7 @@ SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol);
** CAPI3REF: Destroy A Prepared Statement Object
**
** ^The sqlite3_finalize() function is called to delete a [prepared statement].
** ^If the most recent evaluation of the statement encountered no errors or
** ^If the most recent evaluation of the statement encountered no errors
** or if the statement is never been evaluated, then sqlite3_finalize() returns
** SQLITE_OK. ^If the most recent evaluation of statement S failed, then
** sqlite3_finalize(S) returns the appropriate [error code] or
@ -4610,6 +4768,11 @@ struct sqlite3_module {
void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
void **ppArg);
int (*xRename)(sqlite3_vtab *pVtab, const char *zNew);
/* The methods above are in version 1 of the sqlite_module object. Those
** below are for version 2 and greater. */
int (*xSavepoint)(sqlite3_vtab *pVTab, int);
int (*xRelease)(sqlite3_vtab *pVTab, int);
int (*xRollbackTo)(sqlite3_vtab *pVTab, int);
};
/*
@ -5292,7 +5455,7 @@ struct sqlite3_mutex_methods {
**
** ^If the argument to sqlite3_mutex_held() is a NULL pointer then
** the routine should return 1. This seems counter-intuitive since
** clearly the mutex cannot be held if it does not exist. But the
** clearly the mutex cannot be held if it does not exist. But
** the reason the mutex does not exist is because the build is not
** using mutexes. And we do not want the assert() containing the
** call to sqlite3_mutex_held() to fail, so a non-zero return is
@ -5415,7 +5578,8 @@ SQLITE_API int sqlite3_test_control(int op, ...);
#define SQLITE_TESTCTRL_ISKEYWORD 16
#define SQLITE_TESTCTRL_PGHDRSZ 17
#define SQLITE_TESTCTRL_SCRATCHMALLOC 18
#define SQLITE_TESTCTRL_LAST 18
#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19
#define SQLITE_TESTCTRL_LAST 19
/*
** CAPI3REF: SQLite Runtime Status
@ -5424,7 +5588,7 @@ SQLITE_API int sqlite3_test_control(int op, ...);
** about the performance of SQLite, and optionally to reset various
** highwater marks. ^The first argument is an integer code for
** the specific parameter to measure. ^(Recognized integer codes
** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^
** are of the form [status parameters | SQLITE_STATUS_...].)^
** ^The current value of the parameter is returned into *pCurrent.
** ^The highest recorded value is returned in *pHighwater. ^If the
** resetFlag is true, then the highest record value is reset after
@ -5451,12 +5615,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
/*
** CAPI3REF: Status Parameters
** KEYWORDS: {status parameters}
**
** These integer constants designate various run-time status parameters
** that can be returned by [sqlite3_status()].
**
** <dl>
** ^(<dt>SQLITE_STATUS_MEMORY_USED</dt>
** [[SQLITE_STATUS_MEMORY_USED]] ^(<dt>SQLITE_STATUS_MEMORY_USED</dt>
** <dd>This parameter is the current amount of memory checked out
** using [sqlite3_malloc()], either directly or indirectly. The
** figure includes calls made to [sqlite3_malloc()] by the application
@ -5466,23 +5631,24 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
** this parameter. The amount returned is the sum of the allocation
** sizes as reported by the xSize method in [sqlite3_mem_methods].</dd>)^
**
** ^(<dt>SQLITE_STATUS_MALLOC_SIZE</dt>
** [[SQLITE_STATUS_MALLOC_SIZE]] ^(<dt>SQLITE_STATUS_MALLOC_SIZE</dt>
** <dd>This parameter records the largest memory allocation request
** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their
** internal equivalents). Only the value returned in the
** *pHighwater parameter to [sqlite3_status()] is of interest.
** The value written into the *pCurrent parameter is undefined.</dd>)^
**
** ^(<dt>SQLITE_STATUS_MALLOC_COUNT</dt>
** [[SQLITE_STATUS_MALLOC_COUNT]] ^(<dt>SQLITE_STATUS_MALLOC_COUNT</dt>
** <dd>This parameter records the number of separate memory allocations
** currently checked out.</dd>)^
**
** ^(<dt>SQLITE_STATUS_PAGECACHE_USED</dt>
** [[SQLITE_STATUS_PAGECACHE_USED]] ^(<dt>SQLITE_STATUS_PAGECACHE_USED</dt>
** <dd>This parameter returns the number of pages used out of the
** [pagecache memory allocator] that was configured using
** [SQLITE_CONFIG_PAGECACHE]. The
** value returned is in pages, not in bytes.</dd>)^
**
** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]]
** ^(<dt>SQLITE_STATUS_PAGECACHE_OVERFLOW</dt>
** <dd>This parameter returns the number of bytes of page cache
** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE]
@ -5492,13 +5658,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because
** no space was left in the page cache.</dd>)^
**
** ^(<dt>SQLITE_STATUS_PAGECACHE_SIZE</dt>
** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(<dt>SQLITE_STATUS_PAGECACHE_SIZE</dt>
** <dd>This parameter records the largest memory allocation request
** handed to [pagecache memory allocator]. Only the value returned in the
** *pHighwater parameter to [sqlite3_status()] is of interest.
** The value written into the *pCurrent parameter is undefined.</dd>)^
**
** ^(<dt>SQLITE_STATUS_SCRATCH_USED</dt>
** [[SQLITE_STATUS_SCRATCH_USED]] ^(<dt>SQLITE_STATUS_SCRATCH_USED</dt>
** <dd>This parameter returns the number of allocations used out of the
** [scratch memory allocator] configured using
** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not
@ -5506,7 +5672,7 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
** outstanding at time, this parameter also reports the number of threads
** using scratch memory at the same time.</dd>)^
**
** ^(<dt>SQLITE_STATUS_SCRATCH_OVERFLOW</dt>
** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(<dt>SQLITE_STATUS_SCRATCH_OVERFLOW</dt>
** <dd>This parameter returns the number of bytes of scratch memory
** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH]
** buffer and where forced to overflow to [sqlite3_malloc()]. The values
@ -5516,13 +5682,13 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
** slots were available.
** </dd>)^
**
** ^(<dt>SQLITE_STATUS_SCRATCH_SIZE</dt>
** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(<dt>SQLITE_STATUS_SCRATCH_SIZE</dt>
** <dd>This parameter records the largest memory allocation request
** handed to [scratch memory allocator]. Only the value returned in the
** *pHighwater parameter to [sqlite3_status()] is of interest.
** The value written into the *pCurrent parameter is undefined.</dd>)^
**
** ^(<dt>SQLITE_STATUS_PARSER_STACK</dt>
** [[SQLITE_STATUS_PARSER_STACK]] ^(<dt>SQLITE_STATUS_PARSER_STACK</dt>
** <dd>This parameter records the deepest parser stack. It is only
** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].</dd>)^
** </dl>
@ -5547,9 +5713,9 @@ SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetF
** about a single [database connection]. ^The first argument is the
** database connection object to be interrogated. ^The second argument
** is an integer constant, taken from the set of
** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that
** [SQLITE_DBSTATUS options], that
** determines the parameter to interrogate. The set of
** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely
** [SQLITE_DBSTATUS options] is likely
** to grow in future releases of SQLite.
**
** ^The current value of the requested parameter is written into *pCur
@ -5566,6 +5732,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
/*
** CAPI3REF: Status Parameters for database connections
** KEYWORDS: {SQLITE_DBSTATUS options}
**
** These constants are the available integer "verbs" that can be passed as
** the second argument to the [sqlite3_db_status()] interface.
@ -5577,15 +5744,16 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** if a discontinued or unsupported verb is invoked.
**
** <dl>
** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_USED</dt>
** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_USED</dt>
** <dd>This parameter returns the number of lookaside memory slots currently
** checked out.</dd>)^
**
** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
** <dd>This parameter returns the number malloc attempts that were
** satisfied using lookaside memory. Only the high-water value is meaningful;
** the current value is always zero.)^
**
** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]]
** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt>
** <dd>This parameter returns the number malloc attempts that might have
** been satisfied using lookaside memory but failed due to the amount of
@ -5593,6 +5761,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** Only the high-water value is meaningful;
** the current value is always zero.)^
**
** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]]
** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt>
** <dd>This parameter returns the number malloc attempts that might have
** been satisfied using lookaside memory but failed due to all lookaside
@ -5600,12 +5769,12 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** Only the high-water value is meaningful;
** the current value is always zero.)^
**
** ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
** [[SQLITE_DBSTATUS_CACHE_USED]] ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
** <dd>This parameter returns the approximate number of of bytes of heap
** memory used by all pager caches associated with the database connection.)^
** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0.
**
** ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt>
** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt>
** <dd>This parameter returns the approximate number of of bytes of heap
** memory used to store the schema for all databases associated
** with the connection - main, temp, and any [ATTACH]-ed databases.)^
@ -5614,7 +5783,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** [shared cache mode] being enabled.
** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0.
**
** ^(<dt>SQLITE_DBSTATUS_STMT_USED</dt>
** [[SQLITE_DBSTATUS_STMT_USED]] ^(<dt>SQLITE_DBSTATUS_STMT_USED</dt>
** <dd>This parameter returns the approximate number of of bytes of heap
** and lookaside memory used by all prepared statements associated with
** the database connection.)^
@ -5636,7 +5805,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** CAPI3REF: Prepared Statement Status
**
** ^(Each prepared statement maintains various
** [SQLITE_STMTSTATUS_SORT | counters] that measure the number
** [SQLITE_STMTSTATUS counters] that measure the number
** of times it has performed specific operations.)^ These counters can
** be used to monitor the performance characteristics of the prepared
** statements. For example, if the number of table steps greatly exceeds
@ -5647,7 +5816,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r
** ^(This interface is used to retrieve and reset counter values from
** a [prepared statement]. The first argument is the prepared statement
** object to be interrogated. The second argument
** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter]
** is an integer code for a specific [SQLITE_STMTSTATUS counter]
** to be interrogated.)^
** ^The current value of the requested counter is returned.
** ^If the resetFlg is true, then the counter is reset to zero after this
@ -5659,24 +5828,25 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
/*
** CAPI3REF: Status Parameters for prepared statements
** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters}
**
** These preprocessor macros define integer codes that name counter
** values associated with the [sqlite3_stmt_status()] interface.
** The meanings of the various counters are as follows:
**
** <dl>
** <dt>SQLITE_STMTSTATUS_FULLSCAN_STEP</dt>
** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]] <dt>SQLITE_STMTSTATUS_FULLSCAN_STEP</dt>
** <dd>^This is the number of times that SQLite has stepped forward in
** a table as part of a full table scan. Large numbers for this counter
** may indicate opportunities for performance improvement through
** careful use of indices.</dd>
**
** <dt>SQLITE_STMTSTATUS_SORT</dt>
** [[SQLITE_STMTSTATUS_SORT]] <dt>SQLITE_STMTSTATUS_SORT</dt>
** <dd>^This is the number of sort operations that have occurred.
** A non-zero value in this counter may indicate an opportunity to
** improvement performance through careful use of indices.</dd>
**
** <dt>SQLITE_STMTSTATUS_AUTOINDEX</dt>
** [[SQLITE_STMTSTATUS_AUTOINDEX]] <dt>SQLITE_STMTSTATUS_AUTOINDEX</dt>
** <dd>^This is the number of rows inserted into transient indices that
** were created automatically in order to help joins run faster.
** A non-zero value in this counter may indicate an opportunity to
@ -5727,6 +5897,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** the application may discard the parameter after the call to
** [sqlite3_config()] returns.)^
**
** [[the xInit() page cache method]]
** ^(The xInit() method is called once for each effective
** call to [sqlite3_initialize()])^
** (usually only once during the lifetime of the process). ^(The xInit()
@ -5737,6 +5908,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** built-in default page cache is used instead of the application defined
** page cache.)^
**
** [[the xShutdown() page cache method]]
** ^The xShutdown() method is called by [sqlite3_shutdown()].
** It can be used to clean up
** any outstanding resources before process shutdown, if required.
@ -5751,6 +5923,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** ^SQLite will never invoke xInit() more than once without an intervening
** call to xShutdown().
**
** [[the xCreate() page cache methods]]
** ^SQLite invokes the xCreate() method to construct a new cache instance.
** SQLite will typically create one cache instance for each open database file,
** though this is not guaranteed. ^The
@ -5775,6 +5948,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** ^Hence, a cache created with bPurgeable false will
** never contain any unpinned pages.
**
** [[the xCachesize() page cache method]]
** ^(The xCachesize() method may be called at any time by SQLite to set the
** suggested maximum cache-size (number of pages stored by) the cache
** instance passed as the first argument. This is the value configured using
@ -5782,14 +5956,16 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** parameter, the implementation is not required to do anything with this
** value; it is advisory only.
**
** [[the xPagecount() page cache methods]]
** The xPagecount() method must return the number of pages currently
** stored in the cache, both pinned and unpinned.
**
** [[the xFetch() page cache methods]]
** The xFetch() method locates a page in the cache and returns a pointer to
** the page, or a NULL pointer.
** A "page", in this context, means a buffer of szPage bytes aligned at an
** 8-byte boundary. The page to be fetched is determined by the key. ^The
** mimimum key value is 1. After it has been retrieved using xFetch, the page
** minimum key value is 1. After it has been retrieved using xFetch, the page
** is considered to be "pinned".
**
** If the requested page is already in the page cache, then the page cache
@ -5813,6 +5989,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** attempt to unpin one or more cache pages by spilling the content of
** pinned pages to disk and synching the operating system disk cache.
**
** [[the xUnpin() page cache method]]
** ^xUnpin() is called by SQLite with a pointer to a currently pinned page
** as its second argument. If the third parameter, discard, is non-zero,
** then the page must be evicted from the cache.
@ -5825,6 +6002,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** call to xUnpin() unpins the page regardless of the number of prior calls
** to xFetch().
**
** [[the xRekey() page cache methods]]
** The xRekey() method is used to change the key value associated with the
** page passed as the second argument. If the cache
** previously contains an entry associated with newKey, it must be
@ -5837,6 +6015,7 @@ typedef struct sqlite3_pcache sqlite3_pcache;
** of these pages are pinned, they are implicitly unpinned, meaning that
** they can be safely discarded.
**
** [[the xDestroy() page cache method]]
** ^The xDestroy() method is used to delete a cache allocated by xCreate().
** All resources associated with the specified cache should be freed. ^After
** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*]
@ -5899,7 +6078,7 @@ typedef struct sqlite3_backup sqlite3_backup;
** There should be exactly one call to sqlite3_backup_finish() for each
** successful call to sqlite3_backup_init().
**
** <b>sqlite3_backup_init()</b>
** [[sqlite3_backup_init()]] <b>sqlite3_backup_init()</b>
**
** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the
** [database connection] associated with the destination database
@ -5926,7 +6105,7 @@ typedef struct sqlite3_backup sqlite3_backup;
** sqlite3_backup_finish() functions to perform the specified backup
** operation.
**
** <b>sqlite3_backup_step()</b>
** [[sqlite3_backup_step()]] <b>sqlite3_backup_step()</b>
**
** ^Function sqlite3_backup_step(B,N) will copy up to N pages between
** the source and destination databases specified by [sqlite3_backup] object B.
@ -5983,7 +6162,7 @@ typedef struct sqlite3_backup sqlite3_backup;
** by the backup operation, then the backup database is automatically
** updated at the same time.
**
** <b>sqlite3_backup_finish()</b>
** [[sqlite3_backup_finish()]] <b>sqlite3_backup_finish()</b>
**
** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the
** application wishes to abandon the backup operation, the application
@ -6006,7 +6185,8 @@ typedef struct sqlite3_backup sqlite3_backup;
** is not a permanent error and does not affect the return value of
** sqlite3_backup_finish().
**
** <b>sqlite3_backup_remaining(), sqlite3_backup_pagecount()</b>
** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]]
** <b>sqlite3_backup_remaining() and sqlite3_backup_pagecount()</b>
**
** ^Each call to sqlite3_backup_step() sets two values inside
** the [sqlite3_backup] object: the number of pages still to be backed
@ -6392,6 +6572,93 @@ SQLITE_API int sqlite3_wal_checkpoint_v2(
#define SQLITE_CHECKPOINT_FULL 1
#define SQLITE_CHECKPOINT_RESTART 2
/*
** CAPI3REF: Virtual Table Interface Configuration
**
** This function may be called by either the [xConnect] or [xCreate] method
** of a [virtual table] implementation to configure
** various facets of the virtual table interface.
**
** If this interface is invoked outside the context of an xConnect or
** xCreate virtual table method then the behavior is undefined.
**
** At present, there is only one option that may be configured using
** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options
** may be added in the future.
*/
SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...);
/*
** CAPI3REF: Virtual Table Configuration Options
**
** These macros define the various options to the
** [sqlite3_vtab_config()] interface that [virtual table] implementations
** can use to customize and optimize their behavior.
**
** <dl>
** <dt>SQLITE_VTAB_CONSTRAINT_SUPPORT
** <dd>Calls of the form
** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported,
** where X is an integer. If X is zero, then the [virtual table] whose
** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not
** support constraints. In this configuration (which is the default) if
** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire
** statement is rolled back as if [ON CONFLICT | OR ABORT] had been
** specified as part of the users SQL statement, regardless of the actual
** ON CONFLICT mode specified.
**
** If X is non-zero, then the virtual table implementation guarantees
** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before
** any modifications to internal or persistent data structures have been made.
** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite
** is able to roll back a statement or database transaction, and abandon
** or continue processing the current SQL statement as appropriate.
** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns
** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode
** had been ABORT.
**
** Virtual table implementations that are required to handle OR REPLACE
** must do so within the [xUpdate] method. If a call to the
** [sqlite3_vtab_on_conflict()] function indicates that the current ON
** CONFLICT policy is REPLACE, the virtual table implementation should
** silently replace the appropriate rows within the xUpdate callback and
** return SQLITE_OK. Or, if this is not possible, it may return
** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT
** constraint handling.
** </dl>
*/
#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1
/*
** CAPI3REF: Determine The Virtual Table Conflict Policy
**
** This function may only be called from within a call to the [xUpdate] method
** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The
** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL],
** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode
** of the SQL statement that triggered the call to the [xUpdate] method of the
** [virtual table].
*/
SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *);
/*
** CAPI3REF: Conflict resolution modes
**
** These constants are returned by [sqlite3_vtab_on_conflict()] to
** inform a [virtual table] implementation what the [ON CONFLICT] mode
** is for the SQL statement being evaluated.
**
** Note that the [SQLITE_IGNORE] constant is also used as a potential
** return value from the [sqlite3_set_authorizer()] callback and that
** [SQLITE_ABORT] is also a [result code].
*/
#define SQLITE_ROLLBACK 1
/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */
#define SQLITE_FAIL 3
/* #define SQLITE_ABORT 4 // Also an error code */
#define SQLITE_REPLACE 5
/*
** Undo the hack that converts floating point types to integer for

123
util/sectors2sqlite.py Executable file
View File

@ -0,0 +1,123 @@
#!/usr/bin/python3
# Loads block files from sectors folders into map.sqlite database.
# The sectors folder should be safe to remove after this prints "Finished."
import time, os
try:
import sqlite3
except:
exit('You need to have the Python sqlite3 module.')
path = "../world/"
paths = []
# sectors2 gets to try first
if os.path.isdir(path + 'sectors2/'):
paths.append('sectors2')
if os.path.isdir(path + 'sectors/'):
paths.append('sectors')
if not paths:
exit('Could not find sectors folder at ' + path + 'sectors2/ or ' + path + 'sectors/')
def uint(u):
u = int('0x'+u, 16)
return (u if u < 2**11 else u - 2**12)
def int64(u):
while u >= 2**63:
u -= 2**64
while u <= -2**63:
u += 2**64
return u
# Convert sector folder(s) to integer
def getSectorPos(dirname):
if len(dirname) == 8:
# Old layout
x = uint(dirname[:4])
z = uint(dirname[4:])
elif len(dirname) == 7:
# New layout
x = uint(dirname[:3])
z = uint(dirname[4:])
else:
print('Terrible sector at ' + dirname)
return
return x, z
# Convert block file to integer position
def getBlockPos(sectordir, blockfile):
p2d = getSectorPos(sectordir)
if not p2d:
return
if len(blockfile) != 4:
print("Invalid block filename: " + blockfile)
y = uint(blockfile)
return p2d[0], y, p2d[1]
# Convert location to integer
def getBlockAsInteger(p):
return int64(p[2]*16777216 + p[1]*4096 + p[0])
# Init
create = False
if not os.path.isfile(path + 'map.sqlite'):
create = True
conn = sqlite3.connect(path + 'map.sqlite')
if not conn:
exit('Could not open database.')
cur = conn.cursor()
if create:
cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
conn.commit()
print('Created database at ' + path + 'map.sqlite')
# Crawl the folders
count = 0
t = time.time()
for base in paths:
v = 0
if base == 'sectors':
v = 1
elif base == 'sectors2':
v= 2
else:
print('Ignoring base ' + base)
continue
for root, dirs, files in os.walk(path + base):
if files:
for block in files:
pos = getBlockAsInteger(getBlockPos(root[(-8 if v == 1 else -7 if v == 2 else 0):], block))
if pos is None:
print('Ignoring broken path ' + root + '/' + block)
continue
f = open(root+'/'+block, 'rb')
cur.execute('INSERT OR IGNORE INTO `blocks` VALUES(?, ?)', (pos, f.read()))
f.close()
count += 1
if(time.time() - t > 3):
t = time.time()
print(str(count)+' blocks processed...')
conn.commit()
print('Finished. (' + str(count) + ' blocks)')