From 7108a95a00062eb180993e1b11498b734ebd448f Mon Sep 17 00:00:00 2001
From: chenwei_1994 <1606056289@qq.com>
Date: Sun, 12 Jul 2026 14:43:02 +0800
Subject: [PATCH] Bugfix on current version
ext/misc/cksumvfs.c | 11 +-
src/compressvfs.c | 514 +++++++++++++++++++++---------
src/sqlite3.c | 752 ++++++++++++++++++++++++++++++++++----------
3 files changed, 953 insertions(+), 324 deletions(-)
@@ -559,17 +559,17 @@ static int cksmRead(
** (1) the size indicates that we are dealing with a complete
** database page, only support pageSize:4K
** (2) checksum verification is enabled
- ** (3) we are not in the middle of checkpoint
+ ** (3) Skip the WAL log write, it might write segment as iSyncPoint, do checksum at checkpoint
** (4) magic number should be 0xff
** (5) checksum type should be 1, 0 means without checksum
*/
- if( iAmt==4096 /* (1) */
+ if( iAmt==4096 /* (1) */
&& p->verifyCksm /* (2) */
- && !p->inCkpt /* (3) */
+ && !p->isWal /* (3) */
){
if( ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE]!=CKSUMVFS_MAGIC_NUM ){ /* (4) */
sqlite3_log(SQLITE_IOERR_DATA, "unrecognized format, offset %lld of \"%s\", amt:%d", iOfst, p->zFName, iAmt);
- return SQLITE_IOERR_DATA;
+ return rc;
}
if( ((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE+1]==CKSUMVFS_WITHOUT_CHECKSUM ){ /* (5) */
return rc;
@@ -584,7 +584,6 @@ static int cksmRead(
EncodeReservedBytesIntoBase16(cksum, CKSUMVFS_CHECKSUM_SIZE, actual, CKSUM_HEX_LEN);
sqlite3_log(SQLITE_IOERR_DATA, "checksum fault offset %lld of \"%s\", amt:%d, expect:%s, actual:%s",
iOfst, p->zFName, iAmt, expect, actual);
- rc = SQLITE_IOERR_DATA;
}
}
}
@@ -617,7 +616,7 @@ static int cksmWrite(
*/
if( iAmt==4096
&& p->computeCksm
- && !p->inCkpt
+ && !p->isWal
){
((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE]=CKSUMVFS_MAGIC_NUM;
((u8*)zBuf)[iAmt-CKSUMVFS_RESERVED_SIZE+1]=p->verifyCksm ? CKSUMVFS_CALC_CHECKSUM : CKSUMVFS_WITHOUT_CHECKSUM;
@@ -151,6 +151,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */
typedef u32 Pgno;
/* VFS's name */
#define COMPRESS_VFS_NAME "compressvfs"
+#define CKSM_VFS_NAME "cksmvfs"
/* COMPRESSION OPTIONS */
#define COMPRESSION_UNDEFINED 0
@@ -162,6 +163,8 @@ typedef u32 Pgno;
#define SQLITE_SHMMAP_IS_WRITE 0x00000001 /* Flag for xShmMap, extend file if necessary */
#define SQLITE_OPEN_COMPRESS_SHM 0x00010000 /* Flag for xShmMap, need to rename shm file */
+#define SQLITE_WARNING_DUMP (SQLITE_WARNING | (2<<8))
+
/* An open file */
typedef struct{
sqlite3_file base; /* IO methods */
@@ -170,8 +173,10 @@ typedef struct{
u8 bSubDbOpen; /* True to SubDB is opened */
u8 bBegin; /* True to xSync() need commit */
u8 compression; /* Compression options */
- int page_size; /* Uncompressed page size */
+ int pageSize; /* Uncompressed page size */
int persistWalFlag; /* Flag to persist flag */
+ int openFlags; /* Flag to open file */
+ sqlite3_file *pLockFd; /* File handle to lock file */
} CompressFile;
@@ -233,7 +238,24 @@ typedef enum{
BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3,
} BrotliDecoderResult;
-typedef BROTLI_BOOL (*brotliCompress_ptr)(u32, u32, size_t, const u8*, size_t*, u8*);
+/** Options for ::BROTLI_PARAM_MODE parameter. */
+typedef enum BrotliEncoderMde {
+ /**
+ * Default compression mode.
+ *
+ * In this mode compressor does not know anything in advance about the
+ * properties of the input.
+ */
+ BROTLI_MODE_GENERIC = 0,
+ /** Compression mode for UTF-8 formatted text input. */
+ BROTLI_MODE_TEXT = 1,
+ /** Compression mode used in WOFF 2.0 */
+ BROTLI_MODE_FONT = 2
+} BrotliEncoderMde;
+/** Default value for ::BROTLI_PARAM_MODE parameter. */
+#define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC
+
+typedef BROTLI_BOOL (*brotliCompress_ptr)(int, int, BrotliEncoderMde, size_t, const u8*, size_t*, u8*);
typedef BrotliDecoderResult (*brotliDecompress_ptr)(size_t, const u8*, size_t*, u8*);
/*----------------------------brotli header end----------------------------*/
@@ -380,6 +402,7 @@ static int compressBuf(
int ret = brotliCompressPtr(
3, // COMPRESS QUALITY (1-11)
BROTLI_MAX_WINDOW_BITS, // WINDOWS SIZE (10-24)
+ BROTLI_DEFAULT_MODE, // MODE(BROTLI_MODE_GENERIC, TEXT, FONT)
(size_t)src_len,
(const u8 *)src,
&dst_len,
@@ -433,20 +456,32 @@ EXPORT_SYMBOLS int decompressBuf(
return SQLITE_OK;
}
+static int compressConvertErrCode(int errCode){
+ if( errCode==SQLITE_CORRUPT || errCode==SQLITE_BUSY ){
+ return errCode;
+ }
+ return SQLITE_IOERR;
+}
+
/* Check whether the table exists in the OutterDB. */
-static int tableExists(sqlite3 *db, const char *table_name){
+static int tableExists(sqlite3 *db, const char *table_name, u8 *isExist){
sqlite3_stmt *stmt = NULL;
const char *sql = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?;";
- int exists = 0;
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- sqlite3_bind_text(stmt, 1, table_name, -1, SQLITE_STATIC);
- if( sqlite3_step(stmt)==SQLITE_ROW ){
- exists = 1;
- }
- sqlite3_finalize(stmt);
+ int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Check table wrong while prepare stat");
+ return compressConvertErrCode(rc);
}
- return exists;
+ sqlite3_bind_text(stmt, 1, table_name, -1, SQLITE_STATIC);
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
+ sqlite3_log(rc, "Check table wrong while step stat");
+ return compressConvertErrCode(rc);
+ }
+ *isExist = (rc==SQLITE_ROW) ? 1 : 0;
+ return SQLITE_OK;
}
/* Get page size before compressed from OutterDB. */
@@ -457,15 +492,20 @@ static int getCompressPgsize(sqlite3 *db, int *pagesize){
}
sqlite3_stmt *stmt = NULL;
const char *sql = "SELECT pagesize FROM vfs_compression;";
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- if( sqlite3_step(stmt)==SQLITE_ROW ){
- *pagesize = sqlite3_column_int(stmt, 0);
- }else{
- rc = SQLITE_ERROR;
- }
+ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress db prepare to get pgsz wrong");
+ return compressConvertErrCode(rc);
+ }
+ rc = sqlite3_step(stmt);
+ if( rc!=SQLITE_ROW ){
+ sqlite3_log(rc, "Compress db get pgsz wrong, expect at least one row");
sqlite3_finalize(stmt);
+ return compressConvertErrCode(rc);
}
- return rc;
+ *pagesize = sqlite3_column_int(stmt, 0);
+ sqlite3_finalize(stmt);
+ return SQLITE_OK;
}
/* Set page size before compressed to OutterDB. */
@@ -474,50 +514,77 @@ static int setCompressPgsize(sqlite3 *db, int pagesize){
sqlite3_stmt *stmt = NULL;
const char *sql = "UPDATE vfs_compression SET pagesize=?;";
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- sqlite3_bind_int(stmt, 1, pagesize);
- if( sqlite3_step(stmt)!=SQLITE_DONE ){
- rc = SQLITE_ERROR;
- }
- sqlite3_finalize(stmt);
+ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress db prepare update pgsz wrong, size:%d", pagesize);
+ return compressConvertErrCode(rc);
}
- return rc;
+ sqlite3_bind_int(stmt, 1, pagesize);
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ if( rc!=SQLITE_DONE ){
+ sqlite3_log(rc, "Compress db update pgsz wrong, size:%d", pagesize);
+ return compressConvertErrCode(rc);
+ }
+ return SQLITE_OK;
}
/* Get max page number from OutterDB. */
static int getMaxCompressPgno(sqlite3 *db){
sqlite3_stmt *stmt = NULL;
const char *sql = "SELECT MAX(pageno) FROM vfs_pages;";
- int max_pgno = 0;
+ int mxValue = 0;
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- if( sqlite3_step(stmt)==SQLITE_ROW ){
- max_pgno = sqlite3_column_int(stmt, 0);
- }
+ int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "try get max pgno wrong while prepare stat, rc:%d", rc);
+ return mxValue;
+ }
+ rc = sqlite3_step(stmt);
+ if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "try get max pgno wrong while step stat, rc:%d", rc);
sqlite3_finalize(stmt);
+ return mxValue;
}
- return max_pgno;
+ if( rc==SQLITE_ROW ){ // May not exist any pages, if so, return 0, else fetch the value
+ mxValue = sqlite3_column_int(stmt, 0);
+ }
+ sqlite3_finalize(stmt);
+ return mxValue;
}
/* Get Compression option from OutterDB. */
-static void getCompression(sqlite3 *db, CompressFile *pCompress){
+static int getCompression(sqlite3 *db, CompressFile *pCompress){
sqlite3_stmt *stmt = NULL;
const char *sql = "SELECT count(*), compression, pagesize FROM vfs_compression;";
int count = 0;
- pCompress->compression = COMPRESSION_UNDEFINED;
- pCompress->page_size = 0;
-
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- if( sqlite3_step(stmt)==SQLITE_ROW ){
- count = sqlite3_column_int(stmt, 0);
- if( count==1 ){
- pCompress->compression = sqlite3_column_int(stmt, 1);
- pCompress->page_size = sqlite3_column_int(stmt, 2);
- }
- }
+ pCompress->compression = COMPRESSION_ZSTD;
+ pCompress->pageSize = 0;
+
+ int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress db get compression wrong while prepare stat.");
+ return compressConvertErrCode(rc);
+ }
+ rc = sqlite3_step(stmt);
+ if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
+ sqlite3_log(rc, "Compress db get compression wrong while step stat.");
sqlite3_finalize(stmt);
+ return compressConvertErrCode(rc);
}
- return;
+ if( rc==SQLITE_DONE ){
+ sqlite3_finalize(stmt); // No compression options, use default
+ return SQLITE_OK;
+ }
+ count = sqlite3_column_int(stmt, 0);
+ if( count!=1 ){
+ sqlite3_finalize(stmt);
+ return SQLITE_IOERR;
+ }
+ pCompress->compression = sqlite3_column_int(stmt, 1);
+ pCompress->pageSize = sqlite3_column_int(stmt, 2);
+ sqlite3_finalize(stmt);
+ return SQLITE_OK;
}
/*
@@ -531,7 +598,7 @@ static int compressSync(sqlite3_file *pFile, int flags){
if( pCompress->bBegin==1 ){
int rc = sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL);
if( rc!=SQLITE_OK ){
- return SQLITE_IOERR_WRITE;
+ return compressConvertErrCode(rc);
}
pCompress->bBegin = 0;
}
@@ -590,7 +657,9 @@ static int compressDeviceCharacteristics(sqlite3_file *pFile){
*/
static int compressLock(sqlite3_file *pFile, int eFileLock){
assert( pFile );
- return SQLITE_OK;
+ CompressFile *pCompress = (CompressFile *)pFile;
+ sqlite3_file *pSubFile = ORIGFILE(pFile);
+ return pSubFile->pMethods->xLock(pCompress->pLockFd, eFileLock);
}
/*
@@ -598,7 +667,9 @@ static int compressLock(sqlite3_file *pFile, int eFileLock){
*/
static int compressUnlock(sqlite3_file *pFile, int eFileLock){
assert( pFile );
- return SQLITE_OK;
+ CompressFile *pCompress = (CompressFile *)pFile;
+ sqlite3_file *pSubFile = ORIGFILE(pFile);
+ return pSubFile->pMethods->xUnlock(pCompress->pLockFd, eFileLock);
}
/*
@@ -608,11 +679,12 @@ static int compressFileSize(sqlite3_file *pFile, i64 *pSize){
assert( pFile );
CompressFile *pCompress = (CompressFile *)pFile;
sqlite3 *db = pCompress->pDb;
- int rc = getCompressPgsize(db, &pCompress->page_size);
+ int rc = getCompressPgsize(db, &pCompress->pageSize);
if( rc!=SQLITE_OK ){
- return SQLITE_IOERR_FSTAT;
+ sqlite3_log(rc, "compress db get pgsz wrong");
+ return rc;
}
- int pgsize = pCompress->page_size;
+ int pgsize = pCompress->pageSize;
int maxpgno = getMaxCompressPgno(db);
*pSize = (i64)maxpgno * pgsize;
return SQLITE_OK;
@@ -625,31 +697,43 @@ static int compressTruncate(sqlite3_file *pFile, sqlite_int64 size){
assert( pFile );
CompressFile *pCompress = (CompressFile *)pFile;
sqlite3 *db = pCompress->pDb;
- int rc = getCompressPgsize(db, &pCompress->page_size);
- if( rc!=SQLITE_OK || pCompress->page_size==0 ){
+ int rc = getCompressPgsize(db, &pCompress->pageSize);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Get pgsize wrong before truncate pages(%lld)", size);
+ return rc;
+ }
+ int pgsize = pCompress->pageSize;
+ if( pgsize==0 || size%pgsize!=0 ){
return SQLITE_IOERR_TRUNCATE;
}
- int pgsize = pCompress->page_size;
int pgno = size / pgsize;
- if( size % pgsize!=0 || getMaxCompressPgno(db) < pgno ){
- return SQLITE_IOERR_TRUNCATE;
+ int maxPgno = getMaxCompressPgno(db);
+ if( maxPgno<pgno ){
+ sqlite3_log(SQLITE_CORRUPT, "Get max(%d) wrong before truncate pages(%d)", maxPgno, pgno);
+ return SQLITE_CORRUPT;
}
if( pCompress->bBegin!=1 ){
- if( sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL)!=SQLITE_OK ){
- return SQLITE_IOERR_TRUNCATE;
+ rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL);
+ if( rc!=SQLITE_OK ){
+ return compressConvertErrCode(rc);
}
pCompress->bBegin = 1;
}
sqlite3_stmt *stmt = NULL;
const char *sql = "DELETE FROM vfs_pages WHERE pageno > ?;";
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- sqlite3_bind_int(stmt, 1, pgno);
- if( sqlite3_step(stmt)!=SQLITE_DONE ){
- rc = SQLITE_IOERR_TRUNCATE;
- }
- sqlite3_finalize(stmt);
+ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Truncate page wrong while prepare stat, pgno:%d", pgno);
+ return compressConvertErrCode(rc);
}
- return rc;
+ sqlite3_bind_int(stmt, 1, pgno);
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ if( rc!=SQLITE_DONE ){
+ sqlite3_log(rc, "Truncate page wrong while step stat, pgno:%d", pgno);
+ return compressConvertErrCode(rc);
+ }
+ return SQLITE_OK;
}
/*
@@ -661,58 +745,74 @@ static int compressWrite(sqlite3_file *pFile, const void *pBuf, int iAmt, sqlite
assert( iAmt>0 );
CompressFile *pCompress = (CompressFile *)pFile;
sqlite3 *db = pCompress->pDb;
- int rc = getCompressPgsize(db, &pCompress->page_size);
+ int rc = getCompressPgsize(db, &pCompress->pageSize);
if( rc!=SQLITE_OK ){
- return SQLITE_IOERR_WRITE;
+ sqlite3_log(rc, "Missing pgsz(%d), write ofst:%lld, iAmt:%d, flags:%d", pCompress->pageSize,
+ iOfst, iAmt, pCompress->openFlags);
+ return rc;
}
- if( pCompress->page_size<=0 && iAmt >= 512 && iAmt <= 64*1024 && !(iAmt & (iAmt - 1)) ){
+ if( pCompress->pageSize<=0 && iAmt >= 512 && iAmt <= 64*1024 && !(iAmt & (iAmt - 1)) ){
// new compress db need set orignal db's pagesize
rc = setCompressPgsize(db, iAmt);
if( rc!=SQLITE_OK ){
- return SQLITE_IOERR_WRITE;
+ sqlite3_log(rc, "Save page size(%d) wrong, ofst:%lld, flags:%d", iAmt, iOfst, pCompress->openFlags);
+ return rc;
}
- pCompress->page_size = iAmt;
+ pCompress->pageSize = iAmt;
+ }
+ int pgsize = pCompress->pageSize;
+ if( pgsize==0 ){
+ sqlite3_log(SQLITE_IOERR_WRITE, "Zero pageSize in compressWrite, iAmt:%d, iOfst:%lld", iAmt, iOfst);
+ return SQLITE_IOERR_WRITE;
}
- int pgsize = pCompress->page_size;
int pgno = iOfst / pgsize + 1;
if( iAmt!=pgsize || iOfst % pgsize!=0 ){
+ sqlite3_log(SQLITE_IOERR_WRITE, "Mismatch info, iAmt(%d), pgsz(%d), offset:%ld", iAmt, pgsize, iOfst);
return SQLITE_IOERR_WRITE;
}
- int max_compress_size = compressLen(iAmt, pCompress->compression);
- if( max_compress_size<=0 ){
+ int maxSize = compressLen(iAmt, pCompress->compression);
+ if( maxSize<=0 ){
+ sqlite3_log(SQLITE_IOERR_WRITE, "Get compress size(%d) wrong, compression(%d)", maxSize, pCompress->compression);
return SQLITE_IOERR_WRITE;
}
- u8 *compressed_data = sqlite3_malloc(max_compress_size);
- if( compressed_data==NULL ){
+ u8 *tmpData = sqlite3_malloc(maxSize);
+ if( tmpData==NULL ){
+ sqlite3_log(SQLITE_NOMEM, "Malloc size(%d) wrong", maxSize);
return SQLITE_NOMEM;
}
- int compress_data_len = 0;
- if( compressBuf(compressed_data, max_compress_size, &compress_data_len, pBuf, iAmt, pCompress->compression) ){
- sqlite3_free(compressed_data);
+ int len = 0;
+ if( compressBuf(tmpData, maxSize, &len, pBuf, iAmt, pCompress->compression) ){
+ sqlite3_free(tmpData);
+ sqlite3_log(SQLITE_IOERR_WRITE, "Compress buf wrong, pgno(%d), amt(%d), ofst(%lld)", pgno, iAmt, iOfst);
return SQLITE_IOERR_WRITE;
}
if( pCompress->bBegin!=1 ){
- if( sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL)!=SQLITE_OK ){
- sqlite3_free(compressed_data);
- return SQLITE_IOERR_WRITE;
+ if( (rc = sqlite3_exec(db, "BEGIN;", NULL, NULL, NULL))!=SQLITE_OK ){
+ sqlite3_free(tmpData);
+ sqlite3_log(rc, "Begin transaction to insert compressed page wrong, pgno(%d), ofst(%lld)", pgno, iOfst);
+ return compressConvertErrCode(rc);
}
pCompress->bBegin = 1;
}
sqlite3_stmt *stmt = NULL;
const char *sql = "INSERT OR REPLACE INTO vfs_pages(data, pageno) VALUES (?,?);";
- if( sqlite3_prepare_v2(db, sql, -1, &stmt, NULL)==SQLITE_OK ){
- sqlite3_bind_blob(stmt, 1, compressed_data, compress_data_len, SQLITE_STATIC);
- sqlite3_bind_int(stmt, 2, pgno);
- if( sqlite3_step(stmt)!=SQLITE_DONE ){
- sqlite3_free(compressed_data);
- sqlite3_finalize(stmt);
- return SQLITE_IOERR_WRITE;
- }
- sqlite3_finalize(stmt);
+ rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(tmpData);
+ sqlite3_log(rc, "Prepare stat to insert page wrong, pgno(%d), ofst(%lld)", pgno, iOfst);
+ return compressConvertErrCode(rc);
+ }
+ sqlite3_bind_blob(stmt, 1, tmpData, len, SQLITE_STATIC);
+ sqlite3_bind_int(stmt, 2, pgno);
+ rc = sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+ sqlite3_free(tmpData);
+ if( rc!=SQLITE_DONE ){
+ sqlite3_log(rc, "Compress db insert page wrong, pgno(%d), ofst(%lld)", pgno, iOfst);
+ return compressConvertErrCode(rc);
}
- sqlite3_free(compressed_data);
return SQLITE_OK;
}
@@ -731,71 +831,85 @@ static int compressRead(sqlite3_file *pFile, void *pBuf, int iAmt, sqlite_int64
}
(void)memset_s(pBuf, iAmt, 0, iAmt);
sqlite3 *db = pCompress->pDb;
- int rc = getCompressPgsize(db, &pCompress->page_size);
- if( rc!=SQLITE_OK || pCompress->page_size==0 ){
+ int rc = getCompressPgsize(db, &pCompress->pageSize);
+ if( rc!=SQLITE_OK || pCompress->pageSize==0 ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "Missing pgsz(%d), read ofst(%lld), amt(%d), flags(%d)", pCompress->pageSize,
+ iOfst, iAmt, pCompress->openFlags);
return SQLITE_IOERR_SHORT_READ;
}
- int pgsize = pCompress->page_size;
+ int pgsize = pCompress->pageSize;
int pgno = iOfst / pgsize + 1;
int dataidx = iOfst % pgsize;
sqlite3_stmt *stmt = NULL;
const char *sql = "SELECT data, length(data) FROM vfs_pages WHERE pageno=?;";
const void *data = NULL;
- int data_len = 0;
+ int dataLen = 0;
rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);
if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Prepare to get compressed page(%d) wrong, ofst(%lld)", pgno, iOfst);
return SQLITE_CORRUPT;
}
- u8 *decompressed_data = NULL;
+ u8 *decompressedData = NULL;
if( pgsize!=iAmt ){
- decompressed_data = sqlite3_malloc(pgsize);
- if( decompressed_data==NULL ){
+ decompressedData = sqlite3_malloc(pgsize);
+ if( decompressedData==NULL ){
sqlite3_finalize(stmt);
+ sqlite3_log(SQLITE_NOMEM, "Malloc for decompress size(%d) wrong, amt(%d), ofst(%lld)", pgsize, iAmt, iOfst);
return SQLITE_NOMEM;
}
}else{
- decompressed_data = (u8*)pBuf;
+ decompressedData = (u8*)pBuf;
}
- int decompress_data_len = 0;
+ int decompressLen = 0;
sqlite3_bind_int(stmt, 1, pgno);
rc = sqlite3_step(stmt);
if( rc==SQLITE_ROW ){
data = sqlite3_column_blob(stmt, 0);
if( data==NULL ){
rc = SQLITE_IOERR_SHORT_READ;
- goto failed;
+ sqlite3_log(rc, "Get compress page(%d) wrong, empty data, amt(%d), ofst(%lld)", pgno, iAmt, iOfst);
+ goto END_OUT;
}
- data_len = sqlite3_column_int(stmt, 1);
- if( data_len==0 ){
+ dataLen = sqlite3_column_int(stmt, 1);
+ if( dataLen==0 ){
rc = SQLITE_IOERR_SHORT_READ;
- goto failed;
+ sqlite3_log(rc, "Get compress page(%d) wrong, short data, amt(%d), ofst(%lld)", pgno, iAmt, iOfst);
+ goto END_OUT;
}
- if( decompressBuf(decompressed_data, pgsize, &decompress_data_len, data, data_len,
- pCompress->compression)!=SQLITE_OK ){
+ if( decompressBuf(decompressedData, pgsize, &decompressLen, data, dataLen, pCompress->compression)!=SQLITE_OK ){
rc = SQLITE_IOERR_SHORT_READ;
- goto failed;
+ sqlite3_log(rc, "Decompress page(%d) wrong, compression(%d), len(%d), amt(%d), ofst(%lld)", pgno,
+ (int)pCompress->compression, dataLen, iAmt, iOfst);
+ goto END_OUT;
}
- if( decompress_data_len!=pgsize ){
+ if( decompressLen!=pgsize ){
rc = SQLITE_IOERR_SHORT_READ;
- goto failed;
+ sqlite3_log(rc, "Decompress page(%d) wrong, size(%d), pgsz(%d), amt(%d), ofst(%lld)", pgno,
+ decompressLen, pgsize, iAmt, iOfst);
+ goto END_OUT;
}
if( pgsize!=iAmt ){
- rc = memcpy_s(pBuf, iAmt, decompressed_data+dataidx, iAmt);
+ rc = memcpy_s(pBuf, iAmt, decompressedData+dataidx, iAmt);
if( rc!=SQLITE_OK ){
rc = SQLITE_IOERR_SHORT_READ;
- goto failed;
+ sqlite3_log(rc, "Copy decompress page(%d) wrong, size(%d), pgsz(%d), amt(%d), ofst(%lld)", pgno,
+ decompressLen, pgsize, iAmt, iOfst);
+ goto END_OUT;
}
}
rc = SQLITE_OK;
}else if( rc==SQLITE_DONE ){
- rc = SQLITE_IOERR_SHORT_READ;
+ rc = SQLITE_BUSY;
+ sqlite3_log(SQLITE_WARNING_DUMP, "Missing page(%d) while try read, may busy now", pgno);
+ }else if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Got page(%d) from compress db wrong", pgno);
}
- failed:
+END_OUT:
sqlite3_finalize(stmt);
if( pgsize!=iAmt ){
- sqlite3_free(decompressed_data);
+ sqlite3_free(decompressedData);
}
return rc;
@@ -808,22 +922,28 @@ static int compressClose(sqlite3_file *pFile){
assert( pFile );
CompressFile *pCompress = (CompressFile *)pFile;
sqlite3 *db = pCompress->pDb;
+ sqlite3_file *pSubFile = ORIGFILE(pFile);
int rc = compressSync(pFile, 0);
if( rc!=SQLITE_OK ){
- return rc;
+ sqlite3_log(SQLITE_WARNING_DUMP, "Sync wrong while close compress file, rc:%d", rc);
}
if( pCompress->bOutterDbOpen ){
if( db!=NULL ){
rc = sqlite3_close_v2(db);
if( rc!=SQLITE_OK ){
- return rc;
+ sqlite3_log(rc, "compress db close wrong");
+ return compressConvertErrCode(rc);
}
pCompress->pDb = NULL;
}
}
+ if( pCompress->pLockFd ){
+ pSubFile->pMethods->xClose(pCompress->pLockFd);
+ sqlite3_free(pCompress->pLockFd);
+ pCompress->pLockFd = NULL;
+ }
if( pCompress->bSubDbOpen ){
- pFile = ORIGFILE(pFile);
- rc = pFile->pMethods->xClose(pFile);
+ rc = pSubFile->pMethods->xClose(pSubFile);
}
return rc;
}
@@ -888,6 +1008,69 @@ static int compressUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage
return SQLITE_OK;
}
+static int compressOpenLockFile(sqlite3_vfs *pVfs, const char *zName, int flags, sqlite3_file **pFile){
+ const char *walPath = sqlite3_filename_wal(zName);
+ const char *lockPath = walPath + strlen(walPath) + 1; // For compress vfs, lock path place at this position
+ int isLockExist = 0;
+ int rc = ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), lockPath, SQLITE_ACCESS_READWRITE, &isLockExist);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Access lock file failed, path:%s", lockPath);
+ return rc;
+ }
+ if( isLockExist==0 && ((flags&SQLITE_OPEN_CREATE)==0) ){
+ sqlite3_log(SQLITE_WARNING_NOTCOMPRESSDB, "Check lock wrong, compress db should have lock file");
+ return SQLITE_WARNING_NOTCOMPRESSDB;
+ }
+ if( pVfs->szOsFile<=(int)sizeof(CompressFile) ){
+ sqlite3_log(SQLITE_CANTOPEN, "Unexpected file handle size(%d), vfs:%s", pVfs->szOsFile, pVfs->zName);
+ return SQLITE_CANTOPEN;
+ }
+ sqlite3_file *pLockFd = sqlite3_malloc(pVfs->szOsFile - sizeof(CompressFile));
+ if( pLockFd==NULL ){
+ rc = SQLITE_NOMEM;
+ sqlite3_log(rc, "Malloc file handle for lock wrong, size(%d)", (int)(pVfs->szOsFile - sizeof(CompressFile)));
+ return rc;
+ }
+ memset(pLockFd, 0, pVfs->szOsFile - sizeof(CompressFile));
+ rc = ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), lockPath, pLockFd, flags, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress vfs lock file open wrong, path:%s, flag(%d)", lockPath, flags);
+ sqlite3_free(pLockFd);
+ return rc;
+ }
+ *pFile = pLockFd;
+ return rc;
+}
+
+static int compressDbInitCompression(CompressFile *pCompress, sqlite3 *db, u32 compression){
+ const char *pragmaStr = "PRAGMA checksum_persist_enable=ON;PRAGMA page_size=4096;PRAGMA auto_vacuum=INCREMENTAL;";
+ int rc = sqlite3_exec(db, pragmaStr, NULL, NULL, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Config compress db wrong, name");
+ return SQLITE_CANTOPEN;
+ }
+ const char *initCompressionStr = "CREATE TABLE IF NOT EXISTS vfs_compression(compression INTEGER, pagesize INTEGER);";
+ rc = sqlite3_exec(db, initCompressionStr, NULL, NULL, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Configure compress db wrong, create vfs_compression failed");
+ return SQLITE_CANTOPEN;
+ }
+ char compressionSql[COMPRESSION_SQL_MAX_LENGTH] = {0};
+ if( sprintf_s(compressionSql, COMPRESSION_SQL_MAX_LENGTH,
+ "INSERT OR IGNORE INTO vfs_compression(compression, pagesize) VALUES (%u, 0);", compression)<=0 ){
+ sqlite3_log(rc, "Concatenate config stat wrong, compression(%u)", compression);
+ return SQLITE_CANTOPEN;
+ }
+ rc = sqlite3_exec(db, compressionSql, NULL, NULL, NULL);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Config compress db wrong, insert compression(%u) failed", compression);
+ return SQLITE_CANTOPEN;
+ }
+ pCompress->compression = compression;
+ pCompress->pageSize = 0;
+ return SQLITE_OK;
+}
+
/*
** Open a compress file.If this file is not a journal or wal file,
** it will open a OutterDB and create vfs_pages and vfs_compression table
@@ -902,87 +1085,106 @@ static int compressOpen(
){
sqlite3_file *pSubFile = ORIGFILE(pFile);
CompressFile *pCompress = (CompressFile *)pFile;
+ pCompress->openFlags = flags;
int rc = SQLITE_OK;
if( !(flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB)) ){
return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
}
rc = ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pSubFile, flags, pOutFlags);
if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress file open wrong, path:%s, flags:%d", zName, flags);
return rc;
}
pCompress->bSubDbOpen = 1;
sqlite3_int64 fileSize = 0;
+ u8 isExist = 0;
+ sqlite3 *db = NULL;
rc = pSubFile->pMethods->xFileSize(pSubFile, &fileSize);
if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Calculate compress file size wrong, path:%s", zName);
rc = SQLITE_CANTOPEN;
- goto open_end;
+ goto END_OUT;
}
+ rc = compressOpenLockFile(pVfs, zName, flags, &pCompress->pLockFd);
+ if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Compress vfs lock file open wrong, path:%s, flag(%d)", zName, flags);
+ goto END_OUT;
+ }
pFile->pMethods = &compress_io_methods;
- rc = sqlite3_open_v2(zName, &pCompress->pDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, ORIGVFS(pVfs)->zName);
+ rc = sqlite3_open_v2(zName, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, CKSM_VFS_NAME);
if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Open compress db wrong, name:%s", zName);
rc = SQLITE_CANTOPEN;
- goto open_end;
+ goto END_OUT;
}
- pCompress->bOutterDbOpen = 1;
- sqlite3 *db = pCompress->pDb;
- const char *pre_pragma = "PRAGMA page_size=4096;"\
- "PRAGMA auto_vacuum=INCREMENTAL;PRAGMA journal_mode=OFF;";
sqlite3_busy_timeout(db, 2000); // Set time out:2s
- rc = sqlite3_exec(db, pre_pragma, NULL, NULL, NULL);
+ rc = sqlite3_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, NULL);
if( rc!=SQLITE_OK ){
- rc = SQLITE_CANTOPEN;
- goto open_end;
+ sqlite3_log(rc, "Config compress db journal mode wrong, name:%s", zName);
+ goto END_OUT;
+ }
+ pCompress->bOutterDbOpen = 1;
+ rc = tableExists(db, "vfs_compression", &isExist);
+ if( rc!=SQLITE_OK ) {
+ goto END_OUT;
}
- if( tableExists(db, "vfs_compression") ){
- getCompression(db, pCompress);
+ if( isExist ){
+ rc = getCompression(db, pCompress);
+ if( rc!=SQLITE_OK ){
+ goto END_OUT;
+ }
if( pCompress->compression!=COMPRESSION_BROTLI && pCompress->compression!=COMPRESSION_ZSTD ){
rc = SQLITE_CANTOPEN;
- goto open_end;
+ sqlite3_log(rc, "Unrecognized compression(%d), name:%s", pCompress->compression, zName);
+ goto END_OUT;
}
- if( loadCompressAlgorithmExtension(pCompress->compression)==SQLITE_ERROR ){
+ if( loadCompressAlgorithmExtension(pCompress->compression)!=SQLITE_OK ){
rc = SQLITE_CANTOPEN;
- goto open_end;
+ goto END_OUT;
}
}else if( flags&SQLITE_OPEN_MAIN_DB && fileSize!=0 ){
rc = SQLITE_WARNING_NOTCOMPRESSDB;
sqlite3_log(rc, "open compress database go wrong, it should be a compressed db");
- goto open_end;
+ goto END_OUT;
}else{
if( loadCompressAlgorithmExtension(COMPRESSION_UNDEFINED)==SQLITE_ERROR ){
rc = SQLITE_CANTOPEN;
- goto open_end;
- }
- const char *init_compression_sql = "CREATE TABLE vfs_compression (compression INTEGER, pagesize INTEGER);";
- rc = sqlite3_exec(db, init_compression_sql, NULL, NULL, NULL);
- if( rc!=SQLITE_OK ){
- rc = SQLITE_CANTOPEN;
- goto open_end;
- }
- char set_compression_sql[COMPRESSION_SQL_MAX_LENGTH] = {0};
- if( sprintf_s(set_compression_sql, COMPRESSION_SQL_MAX_LENGTH,
- "INSERT INTO vfs_compression(compression, pagesize) VALUES (%u, 0);", g_compress_algo_load)<=0 ){
- rc = SQLITE_CANTOPEN;
- goto open_end;
+ goto END_OUT;
}
- rc = sqlite3_exec(db, set_compression_sql, NULL, NULL, NULL);
+ rc = compressDbInitCompression(pCompress, db, g_compress_algo_load);
if( rc!=SQLITE_OK ){
- rc = SQLITE_CANTOPEN;
- goto open_end;
+ sqlite3_log(rc, "Init compression info wrong, name:%s", zName);
+ goto END_OUT;
}
- pCompress->compression = g_compress_algo_load;
- pCompress->page_size = 0;
}
- if( tableExists(db, "vfs_pages") ){
- goto open_end;
+ rc = tableExists(db, "vfs_pages", &isExist);
+ if( rc!=SQLITE_OK ){
+ goto END_OUT;
+ }
+ if( isExist ){
+ goto END_OUT;
}
- const char *create_sql = "CREATE TABLE vfs_pages (pageno INTEGER PRIMARY KEY, data BLOB NOT NULL);";
- rc = sqlite3_exec(db, create_sql, NULL, NULL, NULL);
+ const char *pageDdlStr = "CREATE TABLE IF NOT EXISTS vfs_pages (pageno INTEGER PRIMARY KEY, data BLOB NOT NULL);";
+ rc = sqlite3_exec(db, pageDdlStr, NULL, NULL, NULL);
if( rc!=SQLITE_OK ){
+ sqlite3_log(rc, "Config compress db wrong, name:%s, create page table failed", zName);
rc = SQLITE_CANTOPEN;
}
- open_end:
+END_OUT:
+ if( rc==SQLITE_OK ){
+ pCompress->pDb = db;
+ }else{
+ if( db!=NULL ){
+ sqlite3_close_v2(db);
+ }
+ if( pCompress->pLockFd ){
+ pSubFile->pMethods->xClose(pCompress->pLockFd);
+ sqlite3_free(pCompress->pLockFd);
+ pCompress->pLockFd = NULL;
+ }
+ }
return rc;
}
@@ -3661,6 +3661,12 @@ SQLITE_API int sqlite3_set_authorizer(
#define SQLITE_COPY 0 /* No longer used */
#define SQLITE_RECURSIVE 33 /* NULL NULL */
+#ifdef SQLITE_ENABLE_BINLOG
+#define SQLITE_SEARCHABLE_DELETE 101 /* NULL NULL */
+#define SQLITE_SEARCHABLE_INSERT 102 /* NULL NULL */
+#define SQLITE_SEARCHABLE_UPDATE 103 /* NULL NULL */
+#endif/* SQLITE_ENABLE_BINLOG */
+
/*
** CAPI3REF: Tracing And Profiling Functions
** METHOD: sqlite3
@@ -5363,7 +5369,7 @@ SQLITE_API int sqlite3_set_droptable_handle(sqlite3*, void (*xFunc)(sqlite3*,con
#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */
#ifdef SQLITE_ENABLE_BINLOG
-SQLITE_API int sqlite3_is_support_binlog(void);
+SQLITE_API int sqlite3_is_support_binlog(const char *notUsed);
SQLITE_API int sqlite3_replay_binlog(sqlite3 *srcDb, sqlite3 *destDb);
@@ -5375,6 +5381,7 @@ typedef struct BinlogSearchResult {
int fileIndex;
sqlite3_uint64 readPos;
char **nameAndValues; // pointer to the row data in record format
+ int *colTypes; // array of SQL column types for each column
} BinlogSearchResult;
typedef struct BinlogSearchResultSet{
@@ -5410,8 +5417,10 @@ SQLITE_API int sqlite3_set_monitor_config_binlog(sqlite3 *srcDb, MonitorTablesCo
SQLITE_API int sqlite3_set_xChange_callback_binlog(sqlite3 *srcDb, void (*xChangeCallback)(const char *dbPath,
char *tableName));
-SQLITE_API int sqlite3_set_json_parse_callback_binlog(sqlite3 *srcDb,
+SQLITE_API int sqlite3_set_json_parse_callback_binlog(sqlite3 *srcDb,
MonitorTablesConfig*(*jsonParseCallback)(const char *dbPath));
+SQLITE_API int sqlite3_free_json_parse_callback_binlog(sqlite3 *srcDb,
+ int(*freeJsonParseCallback)(MonitorTablesConfig *config));
// get data from binlog file
SQLITE_API int sqlite3_get_search_data_binlog(sqlite3 *srcDb, sqlite3 *destDb, BinlogSearchResultSet **rs);
// free result data
@@ -15104,6 +15113,14 @@ typedef INT16_TYPE LogEst;
#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32))
#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+/*
+** Macro SMXV(n) return the maximum value that can be held in variable n,
+** assuming n is a signed integer type. UMXV(n) is similar for unsigned
+** integer types.
+*/
+#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1)
+#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1)
+
/*
** Round up a number to the next larger multiple of 8. This is used
** to force 8-byte alignment on 64-bit architectures.
@@ -17708,8 +17725,9 @@ typedef struct CodecParameter {
#define SQLITE_UUID_BLOB_LENGTH 16
typedef enum {
- ROW = 0,
- ROW_FOR_SEARCH = 2
+ ROW = 1,
+ ROW_FOR_SEARCH = 2,
+ READ_FOR_SEARCH = 3
} Sqlite3BinlogMode;
typedef enum {
@@ -17814,9 +17832,10 @@ typedef struct Sqlite3BinlogConfig {
void (*xChangeCallback)(const char *dbPath, char *tableName);
MonitorTablesConfig*(*jsonParseCallback)(const char *dbPath);
+int(*freeJsonParseCallback)(MonitorTablesConfig *config);
typedef struct BinlogRow {
- int op; // one of the SQLITE_INSERT, SQLITE_UPDATE, SQLITE_DELETE
+ int op; // one of the SQLITE_INSERT, SQLITE_UPDATE, SQLITE_DELETE,SQLITE_SEARCH, SQLITE_DELETE_SEARCH
sqlite3_int64 rowid; // rowid of changed data, -1 for without rowid table
sqlite3_uint64 nData; // size of pData
int nZero; // extra zero bytes at the end of pData
@@ -17856,6 +17875,8 @@ typedef struct Sqlite3BinlogHandle {
void (*xLogFullCallback)(void *pCtx, u16 currentCount, const char *dbPath);
void (*xChangeCallback)(const char *dbPath, char *tabelName);
MonitorTablesConfig* (*jsonParseCallback)(const char *dbPath);
+ int (*freeJsonParseCallback)(MonitorTablesConfig *config);
+ Table* pTable;
BinlogInstanceT *binlogConn;
BinlogApi binlogApi;
u8 isSkipTrigger;
@@ -17920,7 +17941,7 @@ SQLITE_PRIVATE Table *sqlite3BinlogFindTable(BtCursor *pC);
SQLITE_PRIVATE void sqlite3StoreBinlogRowData(Vdbe *p, BtCursor *pCursor, const BinlogRow *pRow);
SQLITE_PRIVATE void sqlite3FreeBinlogRowData(Vdbe *p);
SQLITE_PRIVATE char *sqlite3BinlogGetNthCol(sqlite3 *db, const Table *pTab, const BinlogRow *pRow, int iCol);
-SQLITE_PRIVATE char *sqlite3BinlogGetNthColForSearch(sqlite3 *db, const Table *pTab, const BinlogRow *pRow, int iCol);
+SQLITE_PRIVATE char *sqlite3BinlogGetNthColForSearch(sqlite3 *db, const Table *pTab, const BinlogRow *pRow, int iCol, int *pColType);
/************** Binlog typedef in sqlite3 END ************************************/
#endif
@@ -19154,7 +19175,7 @@ struct AggInfo {
** from source tables rather than from accumulators */
u8 useSortingIdx; /* In direct mode, reference the sorting index rather
** than the source table */
- u16 nSortingColumn; /* Number of columns in the sorting index */
+ u32 nSortingColumn; /* Number of columns in the sorting index */
int sortingIdx; /* Cursor number of the sorting index */
int sortingIdxPTab; /* Cursor number of pseudo-table */
int iFirstReg; /* First register in range for aCol[] and aFunc[] */
@@ -19163,8 +19184,8 @@ struct AggInfo {
Table *pTab; /* Source table */
Expr *pCExpr; /* The original expression */
int iTable; /* Cursor number of the source table */
- i16 iColumn; /* Column number within the source table */
- i16 iSorterColumn; /* Column number in the sorting index */
+ int iColumn; /* Column number within the source table */
+ int iSorterColumn; /* Column number in the sorting index */
} *aCol;
int nColumn; /* Number of used entries in aCol[] */
int nAccumulator; /* Number of columns that show through to the output.
@@ -23985,6 +24006,9 @@ struct Vdbe {
/* begin of binlog related field */
StmtType stmtType; /* Type of binlog statement */
BinlogDMLData *pBinlogDMLData; /* Data for binlog row-based DML statement */
+ char **changedCols;
+ int changedColsCount;
+ int isInsertOrDelete;
const char *pDataTmp; /* Temporary binlog data for op_notfound */
sqlite3_uint64 nDataTmp; /* Size of pDataTmp */
/* end of binlog related field */
@@ -42437,8 +42461,8 @@ static void enableDbFileDelMonitor(int32_t fd)
}
flags |= HMFS_MONITOR_FL;
ret = ioctl(fd, HMFS_IOCTL_HW_SET_FLAGS, &flags);
- if (ret < 0) {
- sqlite3_log(SQLITE_WARNING, "Fd %d enable del monitor go wrong, errno = %d", fd, errno);
+ if (ret < 0 && errno != EACCES) {
+ sqlite3_log(SQLITE_WARNING, "Flags %u fd %d enable del monitor go wrong, errno = %d", flags, fd, errno);
}
}
@@ -45858,6 +45882,24 @@ static int openDirectory(const char *zFilename, int *pFd){
return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname);
}
+#ifdef LOG_DUMP
+static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow);
+static int dirSyncTimeCostAlert(int fd, int fullSync, int dataOnly)
+{
+ sqlite3_int64 startTimeMs = 0;
+ sqlite3_int64 endTimeMs = 0;
+ // unixCurrentTimeInt64 always return OK
+ (void)unixCurrentTimeInt64(0, &startTimeMs);
+ int rc = full_fsync(fd, 0, 0);
+ (void)unixCurrentTimeInt64(0, &endTimeMs);
+ // 1500 ms
+ if (unlikely(endTimeMs >= (startTimeMs + 1500))) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "[SQLite]Dir sync takes %lld ms", endTimeMs - startTimeMs);
+ }
+ return rc;
+}
+#endif
+
/*
** Make sure all writes to a particular file are committed to disk.
**
@@ -45909,7 +45951,11 @@ static int unixSync(sqlite3_file *id, int flags){
HAVE_FULLFSYNC, isFullsync));
rc = osOpenDirectory(pFile->zPath, &dirfd);
if( rc==SQLITE_OK ){
+#ifndef LOG_DUMP
full_fsync(dirfd, 0, 0);
+#else
+ dirSyncTimeCostAlert(dirfd, 0, 0);
+#endif
robust_close(pFile, dirfd, __LINE__);
}else{
assert( rc==SQLITE_CANTOPEN );
@@ -61618,6 +61664,8 @@ static void MetaDwrCheckVacuum(BtShared *pBt);
static int MetaDwrRecoverAndBeginTran(Btree *pBt, int wrflag, int *pSchemaVersion);
static int MetaDwrOpenAndCheck(Btree *pBt);
static void MetaDwrDisable(Btree *pBt);
+static void MetaDwrDumpInfo(Pager *pPager);
+static void MetaDwrCommitUpdate(Pager *pPager);
#define META_HEADER_CHANGED 1
#define META_SCHEMA_CHANGED 2
#define META_IN_RECOVERY 1
@@ -61895,6 +61943,9 @@ struct Pager {
void *metaMapPage;
int (*xGetMethod)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */
#endif
+#ifdef SQLITE_ENABLE_PAGE_COMPRESS
+ char *zLock; /* Point to the lock filename, use to replace main db to lock */
+#endif
};
/*
@@ -63396,11 +63447,12 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){
sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
}
#ifdef SQLITE_META_DWR
- if (bCommit && pPager->metaChanged != 0) {
- sqlite3BeginBenignMalloc();
- (void)MetaDwrWriteHeader(pPager, pPager->metaHdr);
- sqlite3EndBenignMalloc();
- pPager->metaChanged = 0;
+ if (bCommit) {
+ MetaDwrCommitUpdate(pPager);
+ if (pPager->metaChanged != 0) {
+ (void)MetaDwrWriteHeader(pPager, pPager->metaHdr);
+ pPager->metaChanged = 0;
+ }
}
#endif
if( pagerUseWal(pPager) ){
@@ -66189,7 +66241,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
** specific formatting and order of the various filenames, so if the format
** changes here, be sure to change it there as well.
**
- ** Addition, in case of enable page compression, journal&WAL filename will add a file extension:"compress"
+ ** Addition, in case of enable page compression, journal&WAL filename will add a file extension:"compress",
+ ** lock file need an other setting
** So the final layout in memory is as follows:
**
** Pager object (sizeof(Pager) bytes)
@@ -66203,13 +66256,16 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
** URI query parameters (nUriByte bytes)
** Journal filename (nPathname+16+1 bytes)
** WAL filename (nPathname+12+1 bytes)
+ ** Lock filename (nPathname+13+1 bytes)
** \0\0\0 terminator (3 bytes)
**
*/
int fileExtSz = 0;
+ int lockExtSz = 0;
#ifdef SQLITE_ENABLE_PAGE_COMPRESS
if( sqlite3_stricmp(pVfs->zName, "compressvfs")==0 ){
fileExtSz = 8; /* 8 means the size of suffix:"compress" */
+ lockExtSz = 13; /* 13 means the size of suffix:"-lockcompress" */
}
#endif
assert( SQLITE_PTRSIZE==sizeof(Pager*) );
@@ -66225,6 +66281,9 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
nPathname + 8 + fileExtSz + 1 + /* Journal filename */
#ifndef SQLITE_OMIT_WAL
nPathname + 4 + fileExtSz + 1 + /* WAL filename */
+#endif
+#ifdef SQLITE_ENABLE_PAGE_COMPRESS
+ nPathname + lockExtSz + 1 + /* Lock filename */
#endif
3 /* Terminator */
);
@@ -66293,6 +66352,19 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
pPager->zWal = 0;
}
#endif
+
+#ifdef SQLITE_ENABLE_PAGE_COMPRESS
+ if( nPathname>0 ){
+ pPager->zLock = (char*)pPtr;
+ memcpy(pPtr, zPathname, nPathname); pPtr += nPathname;
+ if( lockExtSz>0 ){ /* 13 means the size of string:"-lockcompress" */
+ memcpy(pPtr, "-lockcompress", 13); pPtr += lockExtSz;
+ }
+ pPtr += 1; /* Skip zero suffix */
+ }else{
+ pPager->zLock = 0;
+ }
+#endif
(void)pPtr; /* Suppress warning about unused pPtr value */
if( nPathname ) sqlite3DbFree(0, zPathname);
@@ -77465,7 +77537,13 @@ static void zeroPage(MemPage *pPage, int flags){
data[hdr+7] = 0;
put2byte(&data[hdr+5], pBt->usableSize);
pPage->nFree = (u16)(pBt->usableSize - first);
- decodeFlags(pPage, flags);
+ /*
+ ** pPage might not be a btree page, it might be ovf/ptrmap/free page
+ ** and decodeFlags() return SQLITE_CORRUPT, but no harm is done by this
+ */
+ if( decodeFlags(pPage, flags)==SQLITE_CORRUPT ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "warn: zero page, might be ovf/ptrmap/free page:%d", pPage->pgno);
+ }
pPage->cellOffset = first;
pPage->aDataEnd = &data[pBt->pageSize];
pPage->aCellIdx = &data[first];
@@ -77674,7 +77752,9 @@ static void pageReinit(DbPage *pData){
** But no harm is done by this. And it is very important that
** btreeInitPage() be called on every btree page so we make
** the call for every page that comes in for re-initializing. */
- btreeInitPage(pPage);
+ if( btreeInitPage(pPage)==SQLITE_CORRUPT ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "warn: page reinit, might be ovf/ptrmap/free page:%d", pPage->pgno);
+ }
}
}
}
@@ -78590,6 +78670,11 @@ static int lockBtree(BtShared *pBt){
}
if( nPage>nPageFile ){
if( sqlite3WritableSchema(pBt->db)==0 ){
+ sqlite3_log(SQLITE_WARNING_DUMP, "sqlite3WritableSchema nPage %d nPageFile %d", nPage, nPageFile);
+#ifdef SQLITE_META_DWR
+ DumpLocksByPager(pBt->pPager);
+ MetaDwrDumpInfo(pBt->pPager);
+#endif
rc = SQLITE_CORRUPT_BKPT;
goto page1_init_failed;
}else{
@@ -93152,6 +93237,23 @@ static void vdbeInvokeSqllog(Vdbe *v){
# define vdbeInvokeSqllog(x)
#endif
+#ifdef SQLITE_ENABLE_BINLOG
+SQLITE_PRIVATE void FreeChangedCols(Vdbe *p){
+ sqlite3 *db = p->db;
+ p->isInsertOrDelete = 0;
+ if (p->changedCols==NULL) {
+ return;
+ }
+ for (int i=0; i<p->changedColsCount; i++) {
+ sqlite3DbFree(db, p->changedCols[i]);
+ p->changedCols[i] = NULL;
+ }
+ sqlite3DbFree(db, p->changedCols);
+ p->changedCols = NULL;
+ p->changedColsCount = 0;
+}
+#endif
+
/*
** Clean up a VDBE after execution but do not delete the VDBE just yet.
** Write any error messages into *pzErrMsg. Return the result code.
@@ -93248,6 +93350,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){
}
#endif
#ifdef SQLITE_ENABLE_BINLOG
+ FreeChangedCols(p);
sqlite3FreeBinlogRowData(p);
#endif
return p->rc & db->errMask;
@@ -93354,6 +93457,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
}
#endif
#ifdef SQLITE_ENABLE_BINLOG
+ FreeChangedCols(p);
sqlite3FreeBinlogRowData(p);
#endif
}
@@ -96091,8 +96195,9 @@ SQLITE_API int sqlite3_set_droptable_handle(sqlite3 *db, void (*xFunc)(sqlite3*,
#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */
#ifdef SQLITE_ENABLE_BINLOG
-SQLITE_API int sqlite3_is_support_binlog(void)
+SQLITE_API int sqlite3_is_support_binlog(const char *notUsed)
{
+ (void)notUsed;
return SQLITE_ERROR;
}
@@ -96190,6 +96295,26 @@ SQLITE_API int sqlite3_set_json_parse_callback_binlog(sqlite3 *srcDb,
return SQLITE_OK;
}
+SQLITE_API int sqlite3_free_json_parse_callback_binlog(sqlite3 *srcDb,
+ int(*freeJsonParseCallback)(MonitorTablesConfig *config))
+{
+ if (srcDb == NULL) {
+ sqlite3_log(SQLITE_ERROR, "free json parse binlog parameter is null");
+ return SQLITE_ERROR;
+ }
+ if (!sqlite3SafetyCheckOk(srcDb)) {
+ sqlite3_log(SQLITE_WARNING, "free json parse binlog parameter is not valid db");
+ return SQLITE_MISUSE_BKPT;
+ }
+ if (srcDb->xBinlogHandle.freeJsonParseCallback != NULL) {
+ sqlite3_log(SQLITE_WARNING, "free json parse callback binlog freeJsonParseCallback is alreadySet");
+ return SQLITE_ERROR;
+ }
+ srcDb->xBinlogHandle.freeJsonParseCallback = freeJsonParseCallback;
+ return SQLITE_OK;
+}
+
+
SQLITE_API int sqlite3_free_search_data_binlog(sqlite3 *srcDb, BinlogSearchResultSet **rs)
{
if (srcDb == NULL || rs == NULL || *rs == NULL) {
@@ -96240,9 +96365,13 @@ SQLITE_API int sqlite3_replay_binlog(sqlite3 *srcDb, sqlite3 *destDb)
int rc = SQLITE_OK;
sqlite3_mutex_enter(srcDb->mutex);
rc = sqlite3BinlogReplay(srcDb, destDb);
- if (rc != SQLITE_OK) {
+ if (rc != SQLITE_OK && rc != SQLITE_CORRUPT) {
sqlite3_log(rc, "replay binlog err:%d", rc);
}
+ if (rc == SQLITE_CORRUPT) {
+ rc = SQLITE_ERROR;
+ sqlite3_log(rc, "replay binlog err with not expect way");
+ }
sqlite3_mutex_leave(srcDb->mutex);
return rc;
}
@@ -120799,7 +120928,9 @@ static void findOrCreateAggInfoColumn(
){
struct AggInfo_col *pCol;
int k;
+ int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN];
+ assert( mxTerm <= SMXV(i16) );
assert( pAggInfo->iFirstReg==0 );
pCol = pAggInfo->aCol;
for(k=0; k<pAggInfo->nColumn; k++, pCol++){
@@ -120817,6 +120948,10 @@ static void findOrCreateAggInfoColumn(
assert( pParse->db->mallocFailed );
return;
}
+ if( k>mxTerm ){
+ sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm);
+ k = mxTerm;
+ }
pCol = &pAggInfo->aCol[k];
assert( ExprUseYTab(pExpr) );
pCol->pTab = pExpr->y.pTab;
@@ -120850,6 +120985,7 @@ fix_up_expr:
if( pExpr->op==TK_COLUMN ){
pExpr->op = TK_AGG_COLUMN;
}
+ assert( k <= SMXV(pExpr->iAgg) );
pExpr->iAgg = (i16)k;
}
@@ -120933,13 +121069,19 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
** function that is already in the pAggInfo structure
*/
struct AggInfo_func *pItem = pAggInfo->aFunc;
+ int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN];
+ assert( mxTerm <= SMXV(i16) );
for(i=0; i<pAggInfo->nFunc; i++, pItem++){
if( pItem->pFExpr==pExpr ) break;
if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){
break;
}
}
- if( i>=pAggInfo->nFunc ){
+ if( i>mxTerm ){
+ sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm);
+ i = mxTerm;
+ assert( i<pAggInfo->nFunc );
+ }else if( i>=pAggInfo->nFunc ){
/* pExpr is original. Make a new entry in pAggInfo->aFunc[]
*/
u8 enc = ENC(pParse->db);
@@ -120991,6 +121133,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
*/
assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
ExprSetVVAProperty(pExpr, EP_NoReduce);
+ assert( i <= SMXV(pExpr->iAgg) );
pExpr->iAgg = (i16)i;
pExpr->pAggInfo = pAggInfo;
return WRC_Prune;
@@ -125703,8 +125846,8 @@ static void attachFunc(
if( rc==SQLITE_OK ){
extern int sqlite3CodecAttach(sqlite3*, int, const void*, int);
extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
- int nKey;
- char *zKey;
+ int nKey = 0;
+ char *zKey = NULL;
int t = sqlite3_value_type(argv[2]);
switch( t ){
case SQLITE_INTEGER:
@@ -125721,14 +125864,7 @@ static void attachFunc(
break;
case SQLITE_NULL:
- /* No key specified. Use the key from URI filename, or if none,
- ** use the key from the main database. */
- if( sqlite3CodecQueryParameters(db, zName, zPath)==0 ){
- sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey);
- if( nKey || sqlite3BtreeGetRequestedReserve(db->aDb[0].pBt)>0 ){
- rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey);
- }
- }
+ /* No key specified. just use null key. */
break;
}
}
@@ -142163,6 +142299,8 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_set_xChange_callback_binlog sqlite3_api->set_xChange_callback
/* set binlog json parse callback at client*/
#define sqlite3_set_json_parse_callback_binlog sqlite3_api->set_json_parse_callback
+/* free binlog json parse callback at client*/
+#define sqlite3_free_json_parse_callback_binlog sqlite3_api->free_json_parse_callback
/* get search data from binlog */
#define sqlite3_get_search_data_binlog sqlite3_api->get_search_data
/* free search data from binlog */
@@ -159064,6 +159202,27 @@ SQLITE_PRIVATE void sqlite3Update(
if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb);
+ #ifdef SQLITE_ENABLE_BINLOG
+ if (db->xBinlogHandle.mode == ROW_FOR_SEARCH) {
+ FreeChangedCols(v);
+ int changedColsCount = 0;
+ for (int l = 0; l<pTab->nCol; l++) {
+ if ( aXRef[l]>=0) changedColsCount++;
+ }
+ if (changedColsCount > 0) {
+ v->changedCols = sqlite3DbMallocZero(db, sizeof(char*) * changedColsCount);
+ if (v->changedCols) {
+ int colIndex = 0;
+ for (int l=0; l<pTab->nCol; l++) {
+ if (aXRef[l]>=0) {
+ v->changedCols[colIndex++] = sqlite3DbStrDup(db, pTab->aCol[l].zCnName);
+ }
+ }
+ v->changedColsCount = changedColsCount;
+ }
+ }
+ }
+ #endif
/* Allocate required registers. */
if( !IsVirtual(pTab) ){
/* For now, regRowSet and aRegIdx[nAllIdx] share the same register.
@@ -184567,8 +184726,12 @@ int sqlite3IcuModuleInit(){
sqlite3_log(SQLITE_ERROR, "load icu so failed");
return SQLITE_ERROR;
}
+ MUTEX_LOGIC( sqlite3_mutex *pMainMtx; )
+ MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); )
+ sqlite3_mutex_enter(pMainMtx);
tokenModulePtr = (sqlite3Fts3IcuTokenizerModule_ptr)dlsym(g_library, "sqlite3Fts3IcuTokenizerModule");
icuInitPtr = (sqlite3IcuInit_ptr)dlsym(g_library, "sqlite3IcuInit");
+ sqlite3_mutex_leave(pMainMtx);
if( tokenModulePtr==NULL || icuInitPtr==NULL ){
sqlite3_log(SQLITE_ERROR, "load icu init function failed");
return SQLITE_ERROR;
@@ -184583,7 +184746,17 @@ SQLITE_PRIVATE int sqlite3IcuInitInner(sqlite3 *db)
if( !icuEnable ){
return SQLITE_OK;
}
- return icuInitPtr(db);
+ MUTEX_LOGIC( sqlite3_mutex *pMainMtx; )
+ MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); )
+ sqlite3_mutex_enter(pMainMtx);
+ if ( icuInitPtr == NULL ) {
+ sqlite3_log(SQLITE_ERROR, "icu not load");
+ sqlite3_mutex_leave(pMainMtx);
+ return SQLITE_OK;
+ }
+ int rc = icuInitPtr(db);
+ sqlite3_mutex_leave(pMainMtx);
+ return rc;
}
/************** Continuing where we left off in main.c ***********************/
#endif
@@ -188450,10 +188623,12 @@ opendb_out:
db->eOpenState = SQLITE_STATE_SICK;
}
#ifdef SQLITE_ENABLE_DROPTABLE_CALLBACK
- db->isDropTable = 0;
- db->mDropTableName = NULL;
- db->mDropSchemaName = NULL;
- db->xDropTableHandle = NULL;
+ if( db!=0 ){
+ db->isDropTable = 0;
+ db->mDropTableName = NULL;
+ db->mDropSchemaName = NULL;
+ db->xDropTableHandle = NULL;
+ }
#endif /* SQLITE_ENABLE_DROPTABLE_CALLBACK */
#ifdef SQLITE_ENABLE_BINLOG
sqlite3BinlogReset(db);
@@ -208898,6 +209073,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
return rc;
}
+/*
+** Expression node pExpr is an MSR phrase. This function restarts pExpr
+** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+*/
+int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){
+ int rc = SQLITE_OK;
+ if( pExpr->bEof==0 ){
+ i64 iDocid = pExpr->iDocid;
+ fts3EvalRestart(pCsr, pExpr, &rc);
+ while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){
+ fts3EvalNextRow(pCsr, pExpr, &rc);
+ if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB;
+ }
+ }
+ return rc;
+}
+
+/*
+** If expression pExpr is a phrase expression that uses an MSR query,
+** restart it as a regular, non-incremental query. Return SQLITE_OK
+** if successful, or an SQLite error code otherwise.
+*/
+static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
+ TermOffsetCtx *p = (TermOffsetCtx*)ctx;
+ int rc = SQLITE_OK;
+ if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){
+ rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr);
+ pExpr->pPhrase->bIncr = 0;
+ }
+ return rc;
+}
+
/*
** Implementation of offsets() function.
*/
@@ -208934,6 +209142,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets(
sCtx.iDocid = pCsr->iPrevId;
sCtx.pCsr = pCsr;
+ /* If a query restart will be required, do it here, rather than later of
+ ** after pointers to poslist buffers that may be invalidated by a restart
+ ** have been saved. */
+ rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx);
+ if( rc!=SQLITE_OK ) goto offsets_out;
+
/* Loop through the table columns, appending offset information to
** string-buffer res for each column.
*/
@@ -243644,7 +243858,7 @@ static void fts5DataRelease(Fts5Data *pData){
static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
Fts5Data *pRet = fts5DataRead(p, iRowid);
if( pRet ){
- if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){
+ if( pRet->szLeaf<4 || pRet->szLeaf>pRet->nn ){
p->rc = FTS5_CORRUPT;
fts5DataRelease(pRet);
pRet = 0;
@@ -250163,9 +250377,13 @@ static void fts5IndexIntegrityCheckSegment(
p->rc = FTS5_CORRUPT;
}else{
iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
- res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm));
- if( res==0 ) res = nTerm - nIdxTerm;
- if( res<0 ) p->rc = FTS5_CORRUPT;
+ if( iOff+nTerm>pLeaf->szLeaf ){
+ p->rc = FTS5_CORRUPT;
+ }else{
+ res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm));
+ if( res==0 ) res = nTerm - nIdxTerm;
+ if( res<0 ) p->rc = FTS5_CORRUPT;
+ }
}
fts5IntegrityCheckPgidx(p, pLeaf);
@@ -259128,6 +259346,21 @@ SQLITE_API int sqlite3_stmt_init(
SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }
/************************** End of sqlite3.c ******************************/
+#if SQLITE_OS_UNIX
+#include <unistd.h>
+#include <sys/syscall.h>
+static inline int OsGetTid(void)
+{
+#if defined(__linux__)
+ return (int)syscall(__NR_gettid);
+#elif defined(__APPLE__)
+ return (int)syscall(SYS_thread_selfid);
+#else
+ return 0;
+#endif
+}
+#endif
+
#ifdef SQLITE_CKSUMVFS_STATIC
extern sqlite3_file *cksmvfsGetOrigFile(sqlite3_file *file);
#else
@@ -261762,7 +261995,12 @@ typedef struct MetaDwrHdr {
u32 pageSz;
u32 pageCnt;
u64 dbFileInode;
- u32 reserved[12];
+ u64 walShmFileIno;
+ u64 updateTimeStamp;
+ int updateTid;
+ int updatePid;
+ u32 walFrameCksum[2];
+ u32 reserved[4];
u32 checkSum;
u8 *zones;
Pgno *pages;
@@ -262117,6 +262355,88 @@ static inline u64 CaculateMetaDwrWriteOffset(int pageSz, u32 idx, u8 zone) {
return META_DWR_HEADER_PAGE_SIZE + pageSz * (idx * 2 + zone);
}
+static void MetaDwrDumpInfo(Pager *pPager)
+{
+ MetaDwrHdr *hdr = pPager->metaHdr;
+ if (hdr != NULL) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dbSize %u mxFrameInWal %u dbIno %lu walShmIno %lu",
+ hdr->dbSize, hdr->mxFrameInWal, hdr->dbFileInode, hdr->walShmFileIno);
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr updatestamp %ld updatetid %d updatepid %d",
+ hdr->updateTimeStamp, hdr->updateTid, hdr->updatePid);
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr ckSum0 %u ckSum1 %u", hdr->walFrameCksum[0], hdr->walFrameCksum[1]);
+ }
+#ifndef SQLITE_OMIT_WAL
+ if (!pagerUseWal(pPager)) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr ignore dump wal info");
+ return;
+ }
+ WalIndexHdr *pWalHdr = &pPager->pWal->hdr;
+ sqlite3_log(SQLITE_WARNING_DUMP, "wal ckSum0 %u ckSum1 %u maxFram %u nPage %u", pWalHdr->aFrameCksum[0],
+ pWalHdr->aFrameCksum[1], pWalHdr->mxFrame, pWalHdr->nPage);
+#if SQLITE_OS_UNIX
+ if (hdr == NULL || !hdr->checkFileId) {
+ return;
+ }
+ unixFile *fd = Sqlite3GetUnixFile(pPager->fd, hdr->checkFileId);
+ if (fd == NULL || fd->pInode == NULL) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump invalid db fd");
+ return;
+ }
+ if (fd->pShm == NULL || fd->pShm->pShmNode == NULL || fd->pShm->pShmNode->hShm < 0) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump invalid shm fd");
+ return;
+ }
+ struct stat sStat;
+ if (osFstat(fd->pShm->pShmNode->hShm, &sStat)) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr dump shm stat go wrong %d", errno);
+ return;
+ }
+ sqlite3_log(SQLITE_WARNING_DUMP, "MetaDwr shm ino %lld", (i64)sStat.st_ino);
+#endif
+#endif
+}
+
+static void MetaDwrCommitUpdate(Pager *pPager)
+{
+#if SQLITE_OS_UNIX
+ MetaDwrHdr *hdr = pPager->metaHdr;
+ if (hdr == NULL) {
+ return;
+ }
+ hdr->updateTid = OsGetTid();
+ hdr->updatePid = osGetpid();
+ struct timeval time;
+ gettimeofday(&time, NULL);
+ hdr->updateTimeStamp = time.tv_sec * 1000 + time.tv_usec / 1000;
+#ifndef SQLITE_OMIT_WAL
+ if (pagerUseWal(pPager)) {
+ WalIndexHdr *pWalHdr = &pPager->pWal->hdr;
+ if (pWalHdr->isInit) {
+ hdr->walFrameCksum[0] = pWalHdr->aFrameCksum[0];
+ hdr->walFrameCksum[1] = pWalHdr->aFrameCksum[1];
+ }
+ }
+#endif
+#if !defined(NDEBUG)
+ if (!hdr->checkFileId) {
+ return;
+ }
+ unixFile *fd = Sqlite3GetUnixFile(pPager->fd, hdr->checkFileId);
+ if (fd == NULL || fd->pShm == NULL || fd->pShm->pShmNode == NULL || fd->pShm->pShmNode->hShm < 0) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "update meta header invalid shm fd");
+ return;
+ }
+ struct stat sStat;
+ if (osFstat(fd->pShm->pShmNode->hShm, &sStat)) {
+ hdr->walShmFileIno = 0;
+ sqlite3_log(SQLITE_WARNING_DUMP, "update meta header stat go wrong %d", errno);
+ return;
+ }
+ hdr->walShmFileIno = sStat.st_ino;
+#endif
+#endif
+}
+
static void MetaDwrUpdateHeaderDbInfo(BtShared *pBt) {
MetaDwrHdr *hdr = pBt->pPager->metaHdr;
// 28 offset: dbSize, freelist pageNo, freelist pages count, schema cookie
@@ -262272,7 +262592,10 @@ static int MetaDwrRestoreAllPages(Btree *pBt, const ScanPages *metaPages, MetaDw
return rc;
}
}
- hdr->pageCnt = metaPages->pageCnt;
+ hdr->pageCnt = i;
+ if (metaPages->pageCnt > META_DWR_MAX_PAGES) {
+ sqlite3_log(SQLITE_WARNING_DUMP, "Meta dwr restore pages %u out of range", metaPages->pageCnt);
+ }
MetaDwrUpdateHeaderDbInfo(pBt->pBt);
return rc;
}
@@ -262337,11 +262660,12 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) {
if (pPager->metaMapPage == NULL) {
sqlite3_int64 sz = META_DWR_HEADER_PAGE_SIZE;
sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_CHUNK_SIZE, &sz);
- sqlite3OsFileControlHint(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz);
- void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
- ((unixFile *)metaFd)->h, 0);
- if (page != MAP_FAILED) {
- pPager->metaMapPage = page;
+ if (sqlite3OsFileControl(metaFd, SQLITE_FCNTL_SIZE_HINT, &sz) == SQLITE_OK) {
+ void *page = osMmap(0, META_DWR_HEADER_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
+ ((unixFile *)metaFd)->h, 0);
+ if (page != MAP_FAILED) {
+ pPager->metaMapPage = page;
+ }
}
}
#endif /* SQLITE_OS_UNIX */
@@ -262718,7 +263042,7 @@ CHK_RESTORE_OUT:
static int MetaDwrOpenAndCheck(Btree *pBt)
{
Pager *pPager = pBt->pBt->pPager;
- if (pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) {
+ if (pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) {
return SQLITE_OK;
}
#ifdef SQLITE_HAS_CODEC
@@ -262763,7 +263087,7 @@ DWR_OPEN_OUT:
static void MetaDwrDisable(Btree *pBt)
{
Pager *pPager = pBt->pBt->pPager;
- if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) {
+ if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) {
return;
}
#ifdef SQLITE_HAS_CODEC
@@ -262789,19 +263113,6 @@ static void MetaDwrDisable(Btree *pBt)
#endif /* SQLITE_META_DWR */
#if SQLITE_OS_UNIX
-#include <unistd.h>
-#include <sys/syscall.h>
-static inline int OsGetTid(void)
-{
-#if defined(__linux__)
- return (int)syscall(__NR_gettid);
-#elif defined(__APPLE__)
- return (int)syscall(SYS_thread_selfid);
-#else
- return 0;
-#endif
-}
-
static u8 IsOnlyOneConnection(Pager *pPager)
{
#if SQLITE_OS_UNIX
@@ -262931,8 +263242,13 @@ static inline const char *FlockToName(int l_type)
static int DumpProcessLocks(int fd, struct flock *lock, const char *lockName, char *dumpBuf, int bufLen)
{
+#ifdef F_OFD_GETLK
+ int lkType = F_OFD_GETLK;
+#else
+ int lkType = F_GETLK;
+#endif
dumpBuf[0] = '\0';
- if (osFcntl(fd, F_GETLK, lock) != SQLITE_OK) {
+ if (osFcntl(fd, lkType, lock) != SQLITE_OK) {
sqlite3_log(SQLITE_ERROR, "[SQLite]Get wal file lock ofs %u failed, errno: %d", lock->l_start, errno);
return 0;
}
@@ -263106,6 +263422,35 @@ SQLITE_PRIVATE int sqlite3BinlogInitApi(sqlite3 *db)
return SQLITE_OK;
}
+static void sqlite3FreeMonitorTablesConfig(sqlite3 *db, MonitorTablesConfig *config)
+{
+ if (config == NULL) return;
+
+ if (config->tables != NULL) {
+ for (int i = 0; i < config->tableCount; i++) {
+ MonitorTableCol *tc = &config->tables[i];
+ if (tc->tableName != NULL) {
+ sqlite3DbFree(db, (void *)tc->tableName);
+ tc->tableName = NULL;
+ }
+ if (tc->cols != NULL) {
+ for (int j = 0; j < tc->colCount; j++) {
+ if (tc->cols[j] != NULL) {
+ sqlite3DbFree(db, (void *)tc->cols[j]);
+ tc->cols[j] = NULL;
+ }
+ }
+ sqlite3DbFree(db, tc->cols);
+ tc->cols = NULL;
+ }
+ }
+ sqlite3DbFree(db, config->tables);
+ config->tables = NULL;
+ }
+ sqlite3DbFree(db, config);
+ config = NULL;
+}
+
SQLITE_PRIVATE int sqlite3SetMonitorConfig(sqlite3 *db, MonitorTablesConfig *src)
{
if (src == NULL) {
@@ -263127,7 +263472,10 @@ SQLITE_PRIVATE int sqlite3SetMonitorConfig(sqlite3 *db, MonitorTablesConfig *src
int len = strlen(srcTable->tableName);
dstTable->tableName = sqlite3DbMallocZero(db, len + 1);
if (dstTable->tableName) {
- memcpy((void *)dstTable->tableName, (void *)srcTable->tableName, len);
+ memcpy((void *)dstTable->tableName, (void *)srcTable->tableName, len + 1);
+ } else {
+ sqlite3FreeMonitorTablesConfig(db, dst);
+ return SQLITE_ERROR;
}
}
@@ -263137,17 +263485,19 @@ SQLITE_PRIVATE int sqlite3SetMonitorConfig(sqlite3 *db, MonitorTablesConfig *src
if (dstTable->tableName) {
sqlite3DbFree(db, (void *)dstTable->tableName);
}
- sqlite3DbFree(db, dst->tables);
- sqlite3DbFree(db, dst);
+ sqlite3FreeMonitorTablesConfig(db, dst);
return SQLITE_ERROR;
}
for (int j = 0; j < srcTable->colCount; j++) {
- if (srcTable->cols[j] != NULL) {
+ if (srcTable->cols != NULL && srcTable->cols[j] != NULL) {
int len = strlen(srcTable->cols[j]);
dstTable->cols[j] = sqlite3DbMallocZero(db, len + 1);
if (dstTable->cols[j]) {
- memcpy((void *)dstTable->cols[j], (void *)srcTable->cols[j], len);
+ memcpy((void *)dstTable->cols[j], (void *)srcTable->cols[j], len + 1);
+ } else {
+ sqlite3FreeMonitorTablesConfig(db, dst);
+ return SQLITE_ERROR;
}
}
dstTable->colCount++;
@@ -263160,30 +263510,6 @@ SQLITE_PRIVATE int sqlite3SetMonitorConfig(sqlite3 *db, MonitorTablesConfig *src
return SQLITE_OK;
}
-static void sqlite3FreeMonitorTablesConfig(sqlite3 *db, MonitorTablesConfig *config)
-{
- if (config == NULL) return;
-
- if (config->tables != NULL) {
- for (int i = 0; i < config->tableCount; i++) {
- MonitorTableCol *tc = &config->tables[i];
- if (tc->tableName != NULL) {
- sqlite3DbFree(db, (void *)tc->tableName);
- }
- if (tc->cols != NULL) {
- for (int j = 0; j < tc->colCount; j++) {
- if (tc->cols[j] != NULL) {
- sqlite3DbFree(db, (void *)tc->cols[j]);
- }
- }
- sqlite3DbFree(db, tc->cols);
- }
- }
- sqlite3DbFree(db, config->tables);
- }
- sqlite3DbFree(db, config);
-}
-
SQLITE_PRIVATE int sqlite3SetBinLogConfig(sqlite3 *db, Sqlite3BinlogConfig *bConfig)
{
if (db == NULL) {
@@ -263245,6 +263571,7 @@ SQLITE_PRIVATE void sqlite3BinlogFreeOldColumns(sqlite3 *db, BinlogSearchResult
{
if (oldCounts<=0) {
sqlite3DbFree(db, res->nameAndValues);
+ sqlite3DbFree(db, res->colTypes);
}
for (int i=0; i<oldCounts*2; i++) {
if (res->nameAndValues != NULL && res->nameAndValues[i] != NULL) {
@@ -263260,13 +263587,18 @@ SQLITE_PRIVATE int sqlite3BinlogGetSearchFieldDataForSearch(sqlite3 *db, Table *
if (res->nameAndValues == NULL) {
return SQLITE_ERROR;
}
+ res->colTypes = (int *)sqlite3DbMallocZero(db, pTab->nCol * sizeof(int));
+ if (res->colTypes == NULL) {
+ sqlite3DbFree(db, res->nameAndValues);
+ return SQLITE_ERROR;
+ }
for (int i=0; i<pTab->nCol; i++) {
char *fieldName = sqlite3_mprintf("%s", pTab->aCol[i].zCnName);
if (fieldName == NULL) {
sqlite3BinlogFreeOldColumns(db, res, i-1);
return SQLITE_NOMEM;
}
- char *fieldValue = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, i);
+ char *fieldValue = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, i, &res->colTypes[i]);
if (fieldValue == NULL) {
sqlite3DbFree(db, fieldName);
sqlite3BinlogFreeOldColumns(db, res, i-1);
@@ -263292,18 +263624,23 @@ SQLITE_PRIVATE int sqlite3BinlogGetSearchFieldDataNoRowidForSearch(
if (res->nameAndValues == NULL) {
return SQLITE_ERROR;
}
+ res->colTypes = (int *)sqlite3DbMallocZero(db, pTab->nCol * sizeof(int));
+ if (res->colTypes == NULL) {
+ sqlite3DbFree(db, res->nameAndValues);
+ return SQLITE_ERROR;
+ }
/* Without rowid table fields are rearranged with primary keys at front */
for (int i=0; i<pIdx->nKeyCol; i++) {
i16 iCol = pIdx->aiColumn[i];
res->nameAndValues[i*2] = sqlite3_mprintf("%s", pTab->aCol[iCol].zCnName);
- res->nameAndValues[i*2 + 1] = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, iCol);
+ res->nameAndValues[i*2 + 1] = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, iCol, &res->colTypes[iCol]);
}
int idx = pIdx->nKeyCol;
for (int k=0; k<pTab->nCol; k++) {
if ( (pTab->aCol[k].colFlags & COLFLAG_PRIMKEY) == 0 ) {
res->nameAndValues[idx*2] = sqlite3_mprintf("%s", pTab->aCol[k].zCnName);
- res->nameAndValues[idx*2 + 1] = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, k);
+ res->nameAndValues[idx*2 + 1] = sqlite3BinlogGetNthColForSearch(db, pTab, pRow, k, &res->colTypes[k]);
idx++;
}
}
@@ -263621,7 +263958,7 @@ no_mem:
}
/* Return an insert sql based on the binlog row data, the returned value needs free by caller */
-SQLITE_PRIVATE char *sqlite3BinlogRowInsert(sqlite3 *db, Table *pTab, const BinlogRow *pRow){
+SQLITE_PRIVATE char *sqlite3BinlogRowInsertUpdate(sqlite3 *db, Table *pTab, const BinlogRow *pRow){
char *fieldNames = NULL;
char *fieldValues = NULL;
@@ -263654,24 +263991,6 @@ SQLITE_PRIVATE char *sqlite3BinlogRowInsert(sqlite3 *db, Table *pTab, const Binl
return result;
}
-/* Return an update sql for a regular table. Without rowid table has no update operation */
-SQLITE_PRIVATE char *sqlite3BinlogRowUpdate(sqlite3 *db, Table *pTab, const BinlogRow *pRow){
- char *fieldNames = NULL;
- char *fieldValues = NULL;
- if (sqlite3BinlogGetFieldData(db, pTab, pRow, &fieldNames, &fieldValues) != SQLITE_OK) {
- return NULL;
- }
- char *result = sqlite3_mprintf(
- "insert into \"%s\" (%s, rowid) values (%s, %lld) on conflict(rowid) do update set (%s) = (%s);",
- pTab->zName, fieldNames,
- fieldValues, pRow->rowid,
- fieldNames, fieldValues
- );
- sqlite3DbFree(db, fieldNames);
- sqlite3DbFree(db, fieldValues);
- return result;
-}
-
/* Return the sql statement corresponds to a binlog row data, NULL is returned if error occurs */
SQLITE_PRIVATE char *sqlite3GetBinlogRowStmt(sqlite3 *db, const BinlogRow *pRow, Table *pTab){
assert( db!=NULL );
@@ -263682,9 +264001,8 @@ SQLITE_PRIVATE char *sqlite3GetBinlogRowStmt(sqlite3 *db, const BinlogRow *pRow,
case SQLITE_DELETE:
return HasRowid(pTab) ? sqlite3BinlogRowDelete(pTab, pRow->rowid) : sqlite3BinlogRowDeleteNoRowid(db, pTab, pRow);
case SQLITE_INSERT:
- return sqlite3BinlogRowInsert(db, pTab, pRow);
case SQLITE_UPDATE:
- return sqlite3BinlogRowUpdate(db, pTab, pRow);
+ return sqlite3BinlogRowInsertUpdate(db, pTab, pRow);
default:
return NULL;
}
@@ -263730,49 +264048,85 @@ SQLITE_PRIVATE Table *sqlite3BinlogFindTable(BtCursor *pC){
return pTab;
}
-SQLITE_PRIVATE void CheckAndNotify(sqlite3 *db, char *tableName){
+SQLITE_PRIVATE void NotifyDataChange(Vdbe *p, char *tableName){
+ sqlite3 *db = p->db;
+ const char *zFile = sqlite3_db_filename(db, 0);
+ if (zFile == NULL || (db->xBinlogHandle.flags & BINLOG_FLAG_ENABLE) == 0) {
+ sqlite3_log(SQLITE_ERROR, "binlog db has no file path");
+ }
+ db->xBinlogHandle.xChangeCallback(zFile, tableName);
+}
+
+SQLITE_PRIVATE int CheckNeedNotify(Vdbe *p, char *tableName){
+ sqlite3 *db = p->db;
if (tableName == NULL) {
- return;
+ return 0;
}
- if (db->xBinlogHandle.mode != ROW_FOR_SEARCH) {
- return;
+ char **changedCols = p->changedCols;
+ int changedColsCount = p->changedColsCount;
+ if (changedCols == NULL && changedColsCount > 0) {
+ sqlite3_log(SQLITE_WARNING, "CheckAndNotify: cahngedCiks ss NULL but count > 0");
+ return 0;
+ }
+ if (db->xBinlogHandle.mode != ROW_FOR_SEARCH || (db->xBinlogHandle.flags & BINLOG_FLAG_ENABLE) == 0) {
+ return 0;
}
if (db->xBinlogHandle.monitorConfig==NULL) {
- if (db->xBinlogHandle.jsonParseCallback != NULL) {
- const char *zFile = sqlite3_db_filename(db, 0);
- if (zFile == NULL || (db->xBinlogHandle.flags & BINLOG_FLAG_ENABLE) == 0) {
- sqlite3_log(SQLITE_ERROR, "binlog db has no file path");
- return;
- }
- MonitorTablesConfig *monitorConfig = db->xBinlogHandle.jsonParseCallback(zFile);
- if (monitorConfig == NULL) {
- sqlite3_log(SQLITE_ERROR, "jsonPaarseMonitorConfig get null");
- return;
- }
- int rc = sqlite3SetMonitorConfig(db, monitorConfig);
- if (rc != SQLITE_OK) {
- sqlite3_log(SQLITE_ERROR, "setmonitorConfig fail in jsonPaarseMonitorConfig");
- return;
- }
+ if (db->xBinlogHandle.jsonParseCallback != NULL) {
+ const char *zFile = sqlite3_db_filename(db, 0);
+ if (zFile == NULL) {
+ sqlite3_log(SQLITE_ERROR, "binlog db has no file path");
+ return 0;
+ }
+ MonitorTablesConfig *monitorConfig = db->xBinlogHandle.jsonParseCallback(zFile);
+ if (monitorConfig == NULL) {
+ return 0;
}
+ int rc = sqlite3SetMonitorConfig(db, monitorConfig);
+ if (db->xBinlogHandle.freeJsonParseCallback != NULL) {
+ db->xBinlogHandle.freeJsonParseCallback(monitorConfig);
+ }
+ if (rc != SQLITE_OK) {
+ sqlite3_log(SQLITE_ERROR, "setmonitorConfig fail in jsonPaarseMonitorConfig");
+ return 0;
+ }
+ }
}
if (db->xBinlogHandle.monitorConfig == NULL) {
- return;
+ return 0;
}
MonitorTablesConfig *config = db->xBinlogHandle.monitorConfig;
+ const char *zFile = sqlite3_db_filename(db, 0);
+ if (zFile == NULL) {
+ sqlite3_log(SQLITE_ERROR, "binlog db has no file path");
+ return 0;
+ }
for (int i = 0; i < config->tableCount; i++) {
- if (strcmp(tableName, config->tables[i].tableName) == 0) {
- if (db->xBinlogHandle.xChangeCallback == NULL) {
- return;
- }
- const char *zFile = sqlite3_db_filename(db, 0);
- if (zFile == NULL || (db->xBinlogHandle.flags & BINLOG_FLAG_ENABLE) == 0) {
- sqlite3_log(SQLITE_ERROR, "binlog db has no file path");
- return;
+ if (db->xBinlogHandle.xChangeCallback == NULL) {
+ return 0;
+ }
+ if (p->isInsertOrDelete) {
+ return 1;
+ }
+ int isFound = 0;
+ MonitorTableCol tableConfig = config->tables[i];
+ for (int j = 0; j < changedColsCount; j++) {
+ for (int k = 0; k < tableConfig.colCount; k++) {
+ if (strcmp(changedCols[j], tableConfig.cols[k]) == 0) {
+ isFound = 1;
+ break;
}
- db->xBinlogHandle.xChangeCallback(zFile, tableName);
+ }
+ if (isFound) {
+ break;
+ }
+ }
+ if(!isFound){
+ return 0;
}
+ return 1;
}
+ return 0;
}
SQLITE_PRIVATE int sqlite3BinlogWriteTable(Vdbe *p, Table *pTable){
@@ -263823,12 +264177,28 @@ SQLITE_PRIVATE int sqlite3BinlogDecodeRowBuffer(char *buffer, u32 nBuffer, u8 *o
return SQLITE_OK;
}
-SQLITE_PRIVATE int sqlite3BinlogGetRowBuffer(const BinlogRow *pRow, char **pOutBuffer, u64 *nOutBuffer){
+SQLITE_PRIVATE int sqlite3BinlogGetRowBuffer(const BinlogRow *pRow, u8 isNeedNotify, char **pOutBuffer, u64 *nOutBuffer){
assert( pRow!=NULL );
assert( pOutBuffer!=NULL );
assert( nOutBuffer!=NULL );
- u8 op = pRow->op == SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT;
+ u8 op = pRow->op;
+ if (op != SQLITE_DELETE && op != SQLITE_UPDATE) {
+ op = SQLITE_INSERT;
+ }
+ if (isNeedNotify) {
+ switch (pRow->op) {
+ case SQLITE_DELETE:
+ op = SQLITE_SEARCHABLE_DELETE;
+ break;
+ case SQLITE_INSERT:
+ op = SQLITE_SEARCHABLE_INSERT;
+ break;
+ default:
+ op = SQLITE_SEARCHABLE_UPDATE;
+ break;
+ }
+ }
sqlite3_int64 rowid = pRow->rowid;
sqlite3_uint64 nData = pRow->nData;
int nZero = pRow->nZero;
@@ -263882,7 +264252,7 @@ error_when_memcpy_binlog_row:
return SQLITE_ERROR;
}
-SQLITE_PRIVATE int sqlite3BinlogWriteRowData(Vdbe *p, const BinlogRow *pRow){
+SQLITE_PRIVATE int sqlite3BinlogWriteRowData(Vdbe *p, const BinlogRow *pRow, u8 isNeedNotify){
assert( p!=NULL && pRow!=NULL );
if ((pRow->nData == 0 || pRow->pData == NULL) && pRow->op != SQLITE_DELETE) {
sqlite3_log(SQLITE_ERROR, "binlog row data empty, %llu", pRow->nData);
@@ -263891,7 +264261,7 @@ SQLITE_PRIVATE int sqlite3BinlogWriteRowData(Vdbe *p, const BinlogRow *pRow){
u64 nRowBuffer = 0;
char *pRowBuffer = NULL;
- int rc = sqlite3BinlogGetRowBuffer(pRow, &pRowBuffer, &nRowBuffer);
+ int rc = sqlite3BinlogGetRowBuffer(pRow, isNeedNotify, &pRowBuffer, &nRowBuffer);
if (rc != SQLITE_OK) {
return rc;
}
@@ -263905,7 +264275,8 @@ SQLITE_PRIVATE int sqlite3BinlogWriteRowData(Vdbe *p, const BinlogRow *pRow){
** This is the function called in sqlite3VdbeExec to write a row-based binlog statement.
** If an error occured, then the row data is cleared and the state will be logged in statement mode
**/
-SQLITE_PRIVATE void sqlite3StoreBinlogRowData(Vdbe *p, BtCursor *pCursor, const BinlogRow *pRow){
+SQLITE_PRIVATE void sqlite3StoreBinlogRowData(Vdbe *p, BtCursor *pCursor, const BinlogRow *pRow)
+{
assert( p!=NULL && pCursor!=NULL && pRow!=NULL );
assert( p->db!=NULL );
assert( sqlite3IsRowBasedBinlog(p) );
@@ -263962,8 +264333,15 @@ SQLITE_PRIVATE void sqlite3StoreBinlogRowData(Vdbe *p, BtCursor *pCursor, const
dataContainer->pTable = pTab;
}
- if (sqlite3BinlogWriteRowData(p, pRow) == SQLITE_OK) {
- CheckAndNotify(db, pTab->zName);
+ if (db->xBinlogHandle.mode == ROW_FOR_SEARCH && (pRow->op == SQLITE_INSERT || pRow->op == SQLITE_DELETE)) {
+ p->isInsertOrDelete = 1;
+ }
+ u8 isNeedNotify = CheckNeedNotify(p, pTab->zName);
+ FreeChangedCols(p);
+ if (sqlite3BinlogWriteRowData(p, pRow, isNeedNotify) == SQLITE_OK) {
+ if (isNeedNotify) {
+ NotifyDataChange(p, pTab->zName);
+ }
return;
}
@@ -264103,10 +264481,11 @@ SQLITE_PRIVATE char *sqlite3BinlogGetNthCol(sqlite3 *db, const Table *pTab, cons
}
/* Get the Nth column value for search - returns raw value without SQL quotes */
-SQLITE_PRIVATE char *sqlite3BinlogGetNthColForSearch(sqlite3 *db, const Table *pTab, const BinlogRow *pRow, int iCol) {
+SQLITE_PRIVATE char *sqlite3BinlogGetNthColForSearch(sqlite3 *db, const Table *pTab, const BinlogRow *pRow, int iCol, int *pColType) {
assert( db != NULL && pTab != NULL && pRow != NULL );
assert( iCol >= 0 );
if (HasRowid(pTab) && pTab->iPKey == iCol) {
+ if (pColType) *pColType = SQLITE_INTEGER;
return sqlite3_mprintf("%lld", pRow->rowid);
}
@@ -264130,6 +264509,7 @@ SQLITE_PRIVATE char *sqlite3BinlogGetNthColForSearch(sqlite3 *db, const Table *p
return NULL;
}
int type = sqlite3_value_type(tempVal);
+ if (pColType) *pColType = type;
const unsigned char *temp = sqlite3_value_text(tempVal);
char *res = NULL;
@@ -264157,7 +264537,8 @@ SQLITE_PRIVATE int sqlite3IsSkipWriteBinlog(Vdbe *p) {
|| !isBinlogEnabled
|| p->db->nVdbeExec > 1
|| (p->readOnly && p->stmtType != STMT_TYPE_BEGIN_TRANSACTION && p->stmtType != STMT_TYPE_SAVEPOINT &&
- p->stmtType != STMT_TYPE_COMMIT_TRANSACTION && p->stmtType != STMT_TYPE_ROLLBACK_TRANSACTION);
+ p->stmtType != STMT_TYPE_COMMIT_TRANSACTION && p->stmtType != STMT_TYPE_ROLLBACK_TRANSACTION)
+ || (p->db->xBinlogHandle.mode == READ_FOR_SEARCH);
}
SQLITE_PRIVATE void sqlite3BinlogWrite(Vdbe *p)
@@ -264265,6 +264646,7 @@ SQLITE_PRIVATE void sqlite3BinlogReset(sqlite3 *db)
sqlite3_log(SQLITE_ERROR, "binlog reset parameter is null");
return;
}
+ sqlite3_log(SQLITE_OK, "binlog reset");
db->xBinlogHandle.flags = 0;
(void)memset_s(db->xBinlogHandle.xTid, SQLITE_UUID_BLOB_LENGTH, 0, SQLITE_UUID_BLOB_LENGTH);
db->xBinlogHandle.xErrorCallback = NULL;
@@ -264290,7 +264672,8 @@ SQLITE_PRIVATE void sqlite3BinlogReset(sqlite3 *db)
SQLITE_PRIVATE int sqlite3BinlogResetHwm(sqlite3 *db)
{
if (db == NULL || db->xBinlogHandle.binlogApi.binlogResetSearchHwmApi == NULL) {
- sqlite3_log(SQLITE_ERROR, "binlog reset hwm is null");
+ sqlite3_log(SQLITE_ERROR, "binlog reset hwm is null %d %d", db == NULL,
+ db->xBinlogHandle.binlogApi.binlogResetSearchHwmApi == NULL);
return SQLITE_ERROR;
}
int rc = sqlite3TransferBinlogErrno(db->xBinlogHandle.binlogApi.binlogResetSearchHwmApi(db->xBinlogHandle.binlogConn));
@@ -264360,6 +264743,17 @@ SQLITE_PRIVATE char *sqlite3BinlogGetRowSql(sqlite3 *db, Table* pTable, char *bu
BinlogRow pRow;
pRow.op = op;
+ switch (pRow.op) {
+ case SQLITE_SEARCHABLE_DELETE:
+ pRow.op = SQLITE_DELETE;
+ break;
+ case SQLITE_SEARCHABLE_INSERT:
+ pRow.op = SQLITE_INSERT;
+ break;
+ case SQLITE_SEARCHABLE_UPDATE:
+ pRow.op = SQLITE_UPDATE;
+ break;
+ }
pRow.nData = nData;
pRow.nZero = nZero;
pRow.pData = pData;
@@ -264449,7 +264843,13 @@ SQLITE_PRIVATE int checkIfMonitorTable(sqlite3 *db, char *tableName)
if (config == NULL) {
return 1;
}
+ if (tableName == NULL) {
+ return 0;
+ }
for (int i = 0; i < config->tableCount; i++) {
+ if (config->tables == NULL || config->tables[i].tableName == NULL) {
+ continue;
+ }
if (strcmp(tableName, config->tables[i].tableName) == 0) {
return 1;
}
@@ -264481,15 +264881,20 @@ SQLITE_PRIVATE int sqlite3BinlogGetSearchData(sqlite3 *db, Sqlite3BinlogStmt *bS
rc = SQLITE_ERROR;
sqlite3_log(SQLITE_WARNING, "search binlog find no table, rc=%d", rc);
} else {
+ db->xBinlogHandle.pTable = *pOutTable;
rc = SQLITE_ROW;
}
return rc;
}
case BINLOG_EVENT_TYPE_ROW_FULL_DATA: {
if (pOutTable == NULL || *pOutTable == NULL) {
- rc = SQLITE_ERROR;
- sqlite3_log(SQLITE_ERROR, "binlog row has no table");
- return rc;
+ if (db->xBinlogHandle.pTable != NULL) {
+ *pOutTable = db->xBinlogHandle.pTable;
+ } else {
+ rc = SQLITE_ERROR;
+ sqlite3_log(SQLITE_ERROR, "binlog row has no table");
+ return rc;
+ }
}
if (!checkIfMonitorTable(db, (*pOutTable)->zName)) {
return SQLITE_ROW;
@@ -264510,6 +264915,19 @@ SQLITE_PRIVATE int sqlite3BinlogGetSearchData(sqlite3 *db, Sqlite3BinlogStmt *bS
BinlogRow pRow;
pRow.op = op;
+ switch (pRow.op) {
+ case SQLITE_SEARCHABLE_DELETE:
+ pRow.op = SQLITE_DELETE;
+ break;
+ case SQLITE_SEARCHABLE_INSERT:
+ pRow.op = SQLITE_INSERT;
+ break;
+ case SQLITE_SEARCHABLE_UPDATE:
+ pRow.op = SQLITE_UPDATE;
+ break;
+ default:
+ return SQLITE_ROW;
+ }
pRow.nData = nData;
pRow.nZero = nZero;
pRow.pData = pData;
@@ -264688,6 +265106,11 @@ SQLITE_PRIVATE void BinlogSearchResultFree(sqlite3 *srcDb, BinlogSearchResult *r
sqlite3DbFree(srcDb, row->nameAndValues);
row->nameAndValues = NULL;
}
+
+ if (row->colTypes != NULL) {
+ sqlite3DbFree(srcDb, row->colTypes);
+ row->colTypes = NULL;
+ }
}
/**
@@ -264708,7 +265131,7 @@ SQLITE_PRIVATE void BinlogSearchResultSetDestroy(sqlite3 *srcDb, BinlogSearchRes
SQLITE_PRIVATE int sqlite3BinlogSearchDataGet(sqlite3 *srcDb, sqlite3 *destDb, BinlogSearchResultSet **rs)
{
if ((srcDb->xBinlogHandle.flags & BINLOG_FLAG_ENABLE) == 0) {
- sqlite3_log(SQLITE_ERROR, "binlog replay srcDb not enable binlog");
+ sqlite3_log(SQLITE_ERROR, "binlog replay srcDb not enable binlog %llu", srcDb->xBinlogHandle.flags);
return SQLITE_ERROR;
}
const char *srcFile = sqlite3_db_filename(srcDb, 0);
@@ -264822,12 +265245,13 @@ struct sqlite3_api_routines_extra {
#else
void *dymmyRekeyV3Func;
#endif
- int (*is_support_binlog)(void);
+ int (*is_support_binlog)(const char*);
int (*replay_binlog)(sqlite3*, sqlite3*);
#ifdef SQLITE_ENABLE_BINLOG
int (*set_monitor_config)(sqlite3*, MonitorTablesConfig*);
int (*set_xChange_callback)(sqlite3*, void (*xChangeCallback)(const char *dbPath, char *tableName));
int (*set_json_parse_callback)(sqlite3*, MonitorTablesConfig*(*jsonParseCallback)(const char *dbPath));
+ int (*free_json_parse_callback)(sqlite3*, int(*freeJsonParseCallback)(MonitorTablesConfig *config));
int (*get_search_data_binlog)(sqlite3*, sqlite3*, BinlogSearchResultSet**);
int (*free_search_data_binlog)(sqlite3*, BinlogSearchResultSet**);
int (*set_search_hwm_binlog)(sqlite3*, BinlogSearchHwmT*, BinlogHwmSetModeE);
@@ -264841,6 +265265,8 @@ struct sqlite3_api_routines_extra {
void *dymmyFunc5;
void *dymmyFunc6;
void *dymmyFunc7;
+ void *dymmyFunc8;
+ void *dymmyFunc9;
#endif
#ifdef SQLITE_ENABLE_PAGE_COMPRESS
int (*compressdb_backup)(sqlite3*, const char*);
@@ -264872,6 +265298,7 @@ static const sqlite3_api_routines_extra sqlite3ExtraApis = {
sqlite3_set_monitor_config_binlog,
sqlite3_set_xChange_callback_binlog,
sqlite3_set_json_parse_callback_binlog,
+ sqlite3_free_json_parse_callback_binlog,
sqlite3_get_search_data_binlog,
sqlite3_free_search_data_binlog,
sqlite3_set_search_hwm_binlog,
@@ -264888,6 +265315,7 @@ static const sqlite3_api_routines_extra sqlite3ExtraApis = {
0,
0,
0,
+ 0,
#endif/* SQLITE_ENABLE_BINLOG */
#ifdef SQLITE_ENABLE_PAGE_COMPRESS
sqlite3_compressdb_backup,
--
2.34.1
