diff --git a/Makefile.msc b/Makefile.msc index 561d5f58e0..6f954fb922 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1719,6 +1719,7 @@ FUZZERSHELL_COMPILE_OPTS = FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_DECIMAL_MAX_DIGIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_CARRAY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB @@ -1765,7 +1766,17 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\base64.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\base85.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\completion.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\decimal.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\ieee754.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\regexp.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\series.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\shathree.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\sha1.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\stmtrand.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION @@ -2540,43 +2551,19 @@ coretestprogs: testfixture.exe sqlite3.exe testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe -fulltest: alltest fuzztest - -alltest: $(TESTPROGS) - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\all.test $(TESTOPTS) - -soaktest: $(TESTPROGS) - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS) - -fulltestonly: $(TESTPROGS) fuzztest - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\full.test - -queryplantest: testfixture.exe shell - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) - fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) -# Legacy testing target for third-party integrators. The SQLite -# developers seldom use this target themselves. Instead -# they use "nmake /f Makefile.msc devtest" which runs tests on -# a standard set of options # -test: $(TESTPROGS) sourcetest fuzztest tcltest - -# Minimal testing that runs in less than 3 minutes (on a fast machine) -# -quicktest: testfixture.exe sourcetest - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) - -# This is the common case. Run many tests that do not take too long, -# including fuzzcheck, sqlite3_analyzer, and sqldiff tests. +# Legacy testing targets, no longer used by the developers and +# now aliased to one of the commonly used testing targets. # +quicktest: devtest +test: devtest +fulltest: releasetest +alltest: releasetest +soaktest: releasetest +fulltestonly: releasetest # The veryquick.test TCL tests. # @@ -2660,23 +2647,6 @@ sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) -CHECKER_DEPS =\ - $(TOP)\tool\mkccode.tcl \ - sqlite3.c \ - tclsqlite-ex.c \ - $(TOP)\ext\repair\sqlite3_checker.tcl \ - $(TOP)\ext\repair\checkindex.c \ - $(TOP)\ext\repair\checkfreelist.c \ - $(TOP)\ext\misc\btreeinfo.c \ - $(TOP)\ext\repair\sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ - -sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ - /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) - dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) diff --git a/VERSION b/VERSION index 7ac0b0a686..c5c343b57a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.52.0 +3.53.0 diff --git a/autosetup/README.md b/autosetup/README.md index 2560bfab95..ac013080ad 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -461,4 +461,4 @@ all other significant processing. [sqlite-config.tcl]: /file/autosetup/sqlite-config.tcl [Makefile.in]: /file/Makefile.in [main.mk]: /file/main.mk -[JimTCL]: https://jim.tcl.tk +[JimTCL]: https://msteveb.github.io/jimtcl/ diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index fe1b355ab5..04ef10f0b0 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -1138,6 +1138,27 @@ proc sqlite-check-line-editing {} { set editLibName "readline" ; # "readline" or "editline" set editLibDef "HAVE_READLINE" ; # "HAVE_READLINE" or "HAVE_EDITLINE" set dirLn [opt-val with-linenoise] + + # If none of --with-linenoise, --enable-readline, or --enable-editline + # are provided, but there exists a directory "linenoise" at $HOME or + # a sibling of the build or source directory, then try to use that linenoise + # direcctory. + # + if {"" eq $dirLn + && ![proj-opt-was-provided readline] + && ![proj-opt-was-provided editline] + } { + set dirlist ../linenoise + catch {lappend dirlist [file normalize $::autosetup(srcdir)/../linenoise]} + catch {lappend dirlist $::env(HOME)/linenoise} + foreach d $dirlist { + if {[file exists $d/linenoise.c] && [file exists $d/linenoise.h]} { + set dirLn $d + break + } + } + } + if {"" ne $dirLn} { # Use linenoise from a copy of its sources (not a library)... if {![file isdir $dirLn]} { @@ -1173,6 +1194,8 @@ proc sqlite-check-line-editing {} { if {$::sqliteConfig(use-jim-for-codegen) && 2 == $lnVal} { define-append CFLAGS_JIMSH -DUSE_LINENOISE [get-define CFLAGS_READLINE] user-notice "Adding linenoise support to jimsh." + } else { + msg-result "Using linenoise at [file normalize $dirLn]" } return "linenoise ($flavor)" } elseif {[opt-bool editline]} { diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 556635defa..fea31aae84 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -203,7 +203,16 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define deliberate_fall_through +#if !defined(deliberate_fall_through) +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through +#endif /* ** Macros needed to provide flexible arrays in a portable way diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 439d579366..042fe53946 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -325,7 +325,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->aStat[1].nDoc++; } eState = 2; - /* fall through */ + /* no break */ deliberate_fall_through case 2: if( v==0 ){ /* 0x00. Next integer will be a docid. */ diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 95b33ea318..ee43ca6cca 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -455,7 +455,7 @@ static void fts5SnippetFunction( iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); - aSeen = sqlite3_malloc(nPhrase); + aSeen = sqlite3_malloc64(nPhrase); if( aSeen==0 ){ rc = SQLITE_NOMEM; } diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index afcd83b6ba..d799e34cb4 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -288,7 +288,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ if( nIn<0 ){ nIn = (int)strlen(pIn); } - zRet = (char*)sqlite3_malloc(nIn+1); + zRet = (char*)sqlite3_malloc64((i64)nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index eea82b046d..cea14b500b 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -576,7 +576,7 @@ int sqlite3Fts5ConfigParse( sqlite3_int64 nByte; int bUnindexed = 0; /* True if there are one or more UNINDEXED */ - *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->pGlobal = pGlobal; @@ -1123,5 +1123,3 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_end(ap); } - - diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 352df81f4f..8ecaca34fe 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -314,7 +314,7 @@ int sqlite3Fts5ExprNew( assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ - *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + *ppNew = pNew = sqlite3_malloc64(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); @@ -466,7 +466,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ p2->pRoot = 0; if( sParse.rc==SQLITE_OK ){ - Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64( p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) ); if( ap==0 ){ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index a33dec9a92..ba4a030b7d 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; - *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 4d979b9525..164d613881 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2093,7 +2093,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ - int nNew = pIter->nRowidOffset + 8; + i64 nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; @@ -6418,16 +6418,16 @@ struct Fts5TokenDataMap { ** aMap[] variables. */ struct Fts5TokenDataIter { - int nMapAlloc; /* Allocated size of aMap[] in entries */ - int nMap; /* Number of valid entries in aMap[] */ + i64 nMapAlloc; /* Allocated size of aMap[] in entries */ + i64 nMap; /* Number of valid entries in aMap[] */ Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; /* The following are used for other full-token tokendata queries only. */ - int nIter; - int nIterAlloc; + i64 nIter; + i64 nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[FLEXARRAY]; @@ -6483,11 +6483,11 @@ static void fts5TokendataIterAppendMap( ){ if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nAlloc = nNew * sizeof(Fts5TokenDataMap); + i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + i64 nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + aNew = (Fts5TokenDataMap*)sqlite3_realloc64(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -6513,7 +6513,7 @@ static void fts5TokendataIterAppendMap( */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; - int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap); aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); if( aTmp ){ @@ -7047,9 +7047,10 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ - int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); - Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew; + pNew = (Fts5TokenDataIter*)sqlite3_realloc64(pIn, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; @@ -7146,8 +7147,8 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Ensure the token-mapping is large enough */ if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ - int nNew = (pT->nMapAlloc + nByte) * 2; - Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + i64 nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64( pT->aMap, nNew*sizeof(Fts5TokenDataMap) ); if( aNew==0 ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 9d125095ef..cf033ab5de 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -631,7 +631,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_ERROR; } - idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1); if( idxStr==0 ) return SQLITE_NOMEM; pInfo->idxStr = idxStr; pInfo->needToFreeIdxStr = 1; @@ -3763,7 +3763,7 @@ static int fts5Init(sqlite3 *db){ int rc; Fts5Global *pGlobal = 0; - pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 25cd5c0633..f5d8705ffe 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -391,7 +391,7 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(12, "xSetAuxdata") { - F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); + F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData)); if( pData==0 ){ Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; @@ -780,7 +780,7 @@ static int SQLITE_TCLAPI f5tTokenize( } if( nText>0 ){ - pCopy = sqlite3_malloc(nText); + pCopy = sqlite3_malloc64(nText); if( pCopy==0 ){ tokenizer.xDelete(pTok); Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); @@ -1420,7 +1420,7 @@ static int f5tOrigintextCreate( void *pTokCtx = 0; int rc = SQLITE_OK; - pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer)); if( pTok==0 ){ rc = SQLITE_NOMEM; }else if( nArg<1 ){ @@ -1480,7 +1480,7 @@ static int xOriginToken( int nReq = nToken + 1 + (iEnd-iStart); if( nReq>p->nBuf ){ sqlite3_free(p->aBuf); - p->aBuf = sqlite3_malloc(nReq*2); + p->aBuf = sqlite3_malloc64(nReq*2); if( p->aBuf==0 ) return SQLITE_NOMEM; p->nBuf = nReq*2; } diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index 994d304dc6..c77c49de74 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -194,7 +194,7 @@ static int fts5tokConnectMethod( } if( rc==SQLITE_OK ){ - pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable)); + pTab = (Fts5tokTable*)sqlite3_malloc64(sizeof(Fts5tokTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ @@ -275,7 +275,7 @@ static int fts5tokBestIndexMethod( static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5tokCursor *pCsr; - pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); + pCsr = (Fts5tokCursor *)sqlite3_malloc64(sizeof(Fts5tokCursor)); if( pCsr==0 ){ return SQLITE_NOMEM; } @@ -347,7 +347,7 @@ static int fts5tokCb( if( pCsr->nRow ){ pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); } - pRow->zToken = sqlite3_malloc(nToken+1); + pRow->zToken = sqlite3_malloc64((sqlite3_int64)nToken+1); if( pRow->zToken==0 ) return SQLITE_NOMEM; memcpy(pRow->zToken, pToken, nToken); pRow->zToken[nToken] = 0; @@ -373,8 +373,8 @@ static int fts5tokFilterMethod( fts5tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); + sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index b8a1136465..9908102392 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -72,7 +72,7 @@ static int fts5AsciiCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = sqlite3_malloc(sizeof(AsciiTokenizer)); + p = sqlite3_malloc64(sizeof(AsciiTokenizer)); if( p==0 ){ rc = SQLITE_NOMEM; }else{ @@ -367,7 +367,7 @@ static int fts5UnicodeCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; @@ -590,7 +590,7 @@ static int fts5PorterCreate( zBase = azArg[0]; } - pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); @@ -1297,7 +1297,7 @@ static int fts5TriCreate( rc = SQLITE_ERROR; }else{ int i; - pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 3a6a968f7c..295ace6ba9 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -666,7 +666,7 @@ static int fts5VocabFilterMethod( const char *zCopy = (const char *)sqlite3_value_text(pLe); if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); - pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 4bf120c446..9b2720faf0 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -379,9 +379,6 @@ do_execsql_test 12.2 { db close sqlite3 db test.db -readonly 1 -explain_i { - PRAGMA integrity_check - } do_execsql_test 12.3 { PRAGMA integrity_check } {ok} diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 5f645fae6e..e3fef77637 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -319,7 +319,7 @@ static int intckGetToken(const char *z){ char c = z[0]; int iRet = 1; if( c=='\'' || c=='"' || c=='`' ){ - while( 1 ){ + while( z[iRet] ){ if( z[iRet]==c ){ iRet++; if( z[iRet]!=c ) break; diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 2da767bb0d..28dd74a3b1 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -275,7 +275,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init +int sqlite3_base64_init #else static int sqlite3_base64_init #endif diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 63245e2e4a..52b542bdaa 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -262,7 +262,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER /* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); @@ -282,7 +282,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ return; } } -# endif +#endif /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ @@ -352,14 +352,14 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init +int sqlite3_base85_init #else static int sqlite3_base85_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER { int rc = sqlite3_create_function (db, "is_base85", 1, @@ -367,7 +367,7 @@ static int sqlite3_base85_init 0, is_base85, 0, 0); if( rc!=SQLITE_OK ) return rc; } -# endif +#endif return sqlite3_create_function (db, "base85", 1, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, @@ -432,9 +432,9 @@ int main(int na, char *av[]){ int nc = strlen(cBuf); size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER b85Clean &= allBase85( cBuf, nc ); -# endif +#endif } break; default: diff --git a/ext/misc/completion.c b/ext/misc/completion.c index 67b40d84d1..4b0776064d 100644 --- a/ext/misc/completion.c +++ b/ext/misc/completion.c @@ -199,6 +199,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ completion_cursor *pCur = (completion_cursor*)cur; int eNextPhase = 0; /* Next phase to try if current phase reaches end */ int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + int rc; pCur->iRowid++; while( pCur->ePhase!=COMPLETION_EOF ){ switch( pCur->ePhase ){ @@ -224,22 +225,27 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_TABLES: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str* pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT name FROM \"%w\".sqlite_schema", - zSql, zSep, zDb + zSep, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_COLUMNS; @@ -248,24 +254,29 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_COLUMNS: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str *pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", - zSql, zSep, zDb, zDb + zSep, zDb, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_EOF; @@ -282,9 +293,10 @@ static int completionNext(sqlite3_vtab_cursor *cur){ pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); }else{ /* When all rows are finished, advance to the next phase */ - sqlite3_finalize(pCur->pStmt); + rc = sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; pCur->ePhase = eNextPhase; + if( rc ) return rc; continue; } } diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index f87699f96b..7faa14747e 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -31,6 +31,10 @@ SQLITE_EXTENSION_INIT1 #define IsSpace(X) isspace((unsigned char)X) #endif +#ifndef SQLITE_DECIMAL_MAX_DIGIT +# define SQLITE_DECIMAL_MAX_DIGIT 10000000 +#endif + /* A decimal object */ typedef struct Decimal Decimal; struct Decimal { @@ -69,6 +73,7 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ int i; int iExp = 0; + if( zIn==0 ) goto new_from_text_failed; p = sqlite3_malloc( sizeof(*p) ); if( p==0 ) goto new_from_text_failed; p->sign = 0; @@ -128,9 +133,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -148,9 +154,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; @@ -161,6 +168,7 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ for(i=0; inDigit && p->a[i]==0; i++){} if( i>=p->nDigit ) p->sign = 0; } + if( p->nDigit>SQLITE_DECIMAL_MAX_DIGIT ) goto new_from_text_failed; return p; new_from_text_failed: @@ -291,12 +299,38 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_text(pCtx, z, i, sqlite3_free); } +/* +** Round a decimal value to N significant digits. N must be positive. +*/ +static void decimal_round(Decimal *p, int N){ + int i; + int nZero; + if( N<1 ) return; + if( p==0 ) return; + if( p->nDigit<=N ) return; + for(nZero=0; nZeronDigit && p->a[nZero]==0; nZero++){} + N += nZero; + if( p->nDigit<=N ) return; + if( p->a[N]>4 ){ + p->a[N-1]++; + for(i=N-1; i>0 && p->a[i]>9; i--){ + p->a[i] = 0; + p->a[i-1]++; + } + if( p->a[0]>9 ){ + p->a[0] = 1; + p->nFrac--; + } + } + memset(&p->a[N], 0, p->nDigit - N); +} + /* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ -static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ char *z; /* The output buffer */ int i; /* Loop counter */ int nZero; /* Number of leading zeros */ @@ -314,7 +348,8 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + if( N<1 ) N = 0; + for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} for(nZero=0; nZeroa[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; @@ -430,15 +465,18 @@ static void decimalCmpFunc( static void decimal_expand(Decimal *p, int nDigit, int nFrac){ int nAddSig; int nAddFrac; + signed char *a; if( p==0 ) return; nAddFrac = nFrac - p->nFrac; nAddSig = (nDigit - p->nDigit) - nAddFrac; if( nAddFrac==0 && nAddSig==0 ) return; - p->a = sqlite3_realloc64(p->a, nDigit+1); - if( p->a==0 ){ + if( nDigit+1>SQLITE_DECIMAL_MAX_DIGIT ){ p->oom = 1; return; } + a = sqlite3_realloc64(p->a, nDigit+1); + if( a==0 ){ p->oom = 1; return; } + p->a = a; if( nAddSig ){ memmove(p->a+nAddSig, p->a, p->nDigit); memset(p->a, 0, nAddSig); @@ -533,14 +571,18 @@ static void decimalMul(Decimal *pA, Decimal *pB){ signed char *acc = 0; int i, j, k; int minFrac; + sqlite3_int64 sumDigit; if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + - (sqlite3_int64)pB->nDigit + 2 ); + sumDigit = pA->nDigit; + sumDigit += pB->nDigit; + sumDigit += 2; + if( sumDigit>SQLITE_DECIMAL_MAX_DIGIT ){ pA->oom = 1; return; } + acc = sqlite3_malloc64( sumDigit ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -677,10 +719,16 @@ static void decimalFunc( sqlite3_value **argv ){ Decimal *p = decimal_new(context, argv[0], 0); - UNUSED_PARAMETER(argc); + int N; + if( argc==2 ){ + N = sqlite3_value_int(argv[1]); + if( N>0 ) decimal_round(p, N); + }else{ + N = 0; + } if( p ){ if( sqlite3_user_data(context)!=0 ){ - decimal_result_sci(context, p); + decimal_result_sci(context, p, N); }else{ decimal_result(context, p); } @@ -850,7 +898,7 @@ static void decimalPow2Func( UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA); + decimal_result_sci(context, pA, 0); decimal_free(pA); } } @@ -871,7 +919,9 @@ int sqlite3_decimal_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "decimal", 1, 0, decimalFunc }, + { "decimal", 2, 0, decimalFunc }, { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_exp", 2, 1, decimalFunc }, { "decimal_cmp", 2, 0, decimalCmpFunc }, { "decimal_add", 2, 0, decimalAddFunc }, { "decimal_sub", 2, 0, decimalSubFunc }, diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index db3fd6c3fd..b77c3512bd 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -73,11 +73,6 @@ ** $path is a relative path, then $path is interpreted relative to $dir. ** And the paths returned in the "name" column of the table are also ** relative to directory $dir. -** -** Notes on building this extension for Windows: -** Unless linked statically with the SQLite library, a preprocessor -** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone -** DLL form of this extension for WIN32. See its use below for details. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -94,12 +89,16 @@ SQLITE_EXTENSION_INIT1 # include # include # define STRUCT_STAT struct stat +# include +# include #else # include "windirent.h" # include # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); #endif #include #include @@ -131,12 +130,9 @@ SQLITE_EXTENSION_INIT1 */ #if defined(_WIN32) || defined(WIN32) static int fileio_chmod(const char *zPath, int pmode){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wchmod(b1, pmode); sqlite3_free(b1); return rc; @@ -148,12 +144,9 @@ static int fileio_chmod(const char *zPath, int pmode){ */ #if defined(_WIN32) || defined(WIN32) static int fileio_mkdir(const char *zPath){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wmkdir(b1); sqlite3_free(b1); return rc; @@ -266,50 +259,7 @@ static sqlite3_uint64 fileTimeToUnixTime( return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } - - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -# /* To allow a standalone DLL, use this next replacement function: */ -# undef sqlite3_win32_utf8_to_unicode -# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 -# -LPWSTR utf8_to_utf16(const char *z){ - int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); - LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); - if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) - return rv; - sqlite3_free(rv); - return 0; -} -#endif - -/* -** This function attempts to normalize the time values found in the stat() -** buffer to UTC. This is necessary on Win32, where the runtime library -** appears to return these values as local times. -*/ -static void statTimesToUtc( - const char *zPath, - STRUCT_STAT *pStatBuf -){ - HANDLE hFindFile; - WIN32_FIND_DATAW fd; - LPWSTR zUnicodeName; - extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); - zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); - if( zUnicodeName ){ - memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); - hFindFile = FindFirstFileW(zUnicodeName, &fd); - if( hFindFile!=NULL ){ - pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); - pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); - pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); - FindClose(hFindFile); - } - sqlite3_free(zUnicodeName); - } -} -#endif +#endif /* _WIN32 */ /* ** This function is used in place of stat(). On Windows, special handling @@ -321,14 +271,22 @@ static int fileStat( STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return 1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wstat(b1, pStatBuf); - if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + if( rc==0 ){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(b1, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + } sqlite3_free(b1); return rc; #else @@ -1093,6 +1051,154 @@ static int fsdirRegister(sqlite3 *db){ # define fsdirRegister(x) SQLITE_OK #endif +/* +** This version of realpath() works on any system. The string +** returned is held in memory allocated using sqlite3_malloc64(). +** The caller is responsible for calling sqlite3_free(). +*/ +static char *portable_realpath(const char *zPath){ +#if !defined(_WIN32) /* BEGIN unix */ + + char *zOut = 0; /* Result */ + char *z; /* Temporary buffer */ +#if defined(PATH_MAX) + char zBuf[PATH_MAX+1]; /* Space for the temporary buffer */ +#endif + + if( zPath==0 ) return 0; +#if defined(PATH_MAX) + z = realpath(zPath, zBuf); + if( z ){ + zOut = sqlite3_mprintf("%s", zBuf); + } +#endif /* defined(PATH_MAX) */ + if( zOut==0 ){ + /* Try POSIX.1-2008 malloc behavior */ + z = realpath(zPath, NULL); + if( z ){ + zOut = sqlite3_mprintf("%s", z); + free(z); + } + } + return zOut; + +#else /* End UNIX, Begin WINDOWS */ + + wchar_t *zPath16; /* UTF16 translation of zPath */ + char *zOut = 0; /* Result */ + wchar_t *z = 0; /* Temporary buffer */ + + if( zPath==0 ) return 0; + + zPath16 = sqlite3_win32_utf8_to_unicode(zPath); + if( zPath16==0 ) return 0; + z = _wfullpath(NULL, zPath16, 0); + sqlite3_free(zPath16); + if( z ){ + zOut = sqlite3_win32_unicode_to_utf8(z); + free(z); + } + return zOut; + +#endif /* End WINDOWS, Begin common code */ +} + +/* +** SQL function: realpath(X) +** +** Try to convert file or pathname X into its real, absolute pathname. +** Return NULL if unable. +** +** The file or directory X is not required to exist. The answer is formed +** by calling system realpath() on the prefix of X that does exist and +** appending the tail of X that does not (yet) exist. +*/ +static void realpathFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPath; /* Original input path */ + char *zCopy; /* An editable copy of zPath */ + char *zOut; /* The result */ + char cSep = 0; /* Separator turned into \000 */ + size_t len; /* Prefix length before cSep */ +#ifdef _WIN32 + const int isWin = 1; +#else + const int isWin = 0; +#endif + + (void)argc; + zPath = (const char*)sqlite3_value_text(argv[0]); + if( zPath==0 ) return; + if( zPath[0]==0 ) zPath = "."; + zCopy = sqlite3_mprintf("%s",zPath); + len = strlen(zCopy); + while( len>1 && (zCopy[len-1]=='/' || (isWin && zCopy[len-1]=='\\')) ){ + len--; + } + zCopy[len] = 0; + while( 1 /*exit-by-break*/ ){ + zOut = portable_realpath(zCopy); + zCopy[len] = cSep; + if( zOut ){ + if( cSep ){ + zOut = sqlite3_mprintf("%z%s",zOut,&zCopy[len]); + } + break; + }else{ + size_t i = len-1; + while( i>0 ){ + if( zCopy[i]=='/' || (isWin && zCopy[i]=='\\') ) break; + i--; + } + if( i<=0 ){ + if( zCopy[0]=='/' ){ + zOut = zCopy; + zCopy = 0; + }else if( (zOut = portable_realpath("."))!=0 ){ + zOut = sqlite3_mprintf("%z/%s", zOut, zCopy); + } + break; + } + cSep = zCopy[i]; + zCopy[i] = 0; + len = i; + } + } + sqlite3_free(zCopy); + if( zOut ){ + /* Simplify any "/./" or "/../" that might have snuck into the + ** pathname due to appending of zCopy. We only have to consider + ** unix "/" separators, because the _wfilepath() system call on + ** Windows will have already done this simplification for us. */ + size_t i, j, n; + n = strlen(zOut); + for(i=j=0; i0 && zOut[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + zOut[j++] = zOut[i]; + } + zOut[j] = 0; + + /* Return the result */ + sqlite3_result_text(context, zOut, -1, sqlite3_free); + } +} + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -1119,5 +1225,10 @@ int sqlite3_fileio_init( if( rc==SQLITE_OK ){ rc = fsdirRegister(db); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "realpath", 1, + SQLITE_UTF8, 0, + realpathFunc, 0, 0); + } return rc; } diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index e16d005d9c..3dcf1d667c 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -617,7 +617,7 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; /* Size of output term without nul-term */ + sqlite3_int64 n; /* Size of output term without nul-term */ char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 932b1c278c..f551b2265a 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -259,6 +259,38 @@ static void ieee754func_to_blob( } } +/* +** Functions to convert between 64-bit integers and floats. +** +** The bit patterns are copied. The numeric values are different. +*/ +static void ieee754func_from_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + double r; + sqlite3_int64 v = sqlite3_value_int64(argv[0]); + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(context, r); + } +} +static void ieee754func_to_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[0]); + sqlite3_uint64 v; + memcpy(&v, &r, sizeof(v)); + sqlite3_result_int64(context, v); + } +} + /* ** SQL Function: ieee754_inc(r,N) ** @@ -311,6 +343,8 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_to_int", 1, 0, ieee754func_to_int }, + { "ieee754_from_int", 1, 0, ieee754func_from_int }, { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index e1826caf3c..c5375122f0 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -828,7 +828,6 @@ static void re_bytecode_func( int i; int n; char *z; - (void)argc; static const char *ReOpName[] = { "EOF", "MATCH", @@ -851,6 +850,7 @@ static void re_bytecode_func( "ATSTART", }; + (void)argc; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 02d8649556..fb8f625f51 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -249,16 +249,19 @@ static void sha1Func( SHA1Context cx; int eType = sqlite3_value_type(argv[0]); int nByte = sqlite3_value_bytes(argv[0]); + const unsigned char *pData; char zOut[44]; assert( argc==1 ); if( eType==SQLITE_NULL ) return; hash_init(&cx); if( eType==SQLITE_BLOB ){ - hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_blob(argv[0]); }else{ - hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_text(argv[0]); } + if( pData==0 ) return; + hash_step(&cx, pData, nByte); if( sqlite3_user_data(context)!=0 ){ /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); @@ -320,6 +323,7 @@ static void sha1QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); + if( z==0 ) z = ""; n = (int)strlen(z); hash_step_vformat(&cx,"S%d:",n); hash_step(&cx,(unsigned char*)z,n); diff --git a/ext/misc/sqlar.c b/ext/misc/sqlar.c index 9f726f0b89..873bd6d275 100644 --- a/ext/misc/sqlar.c +++ b/ext/misc/sqlar.c @@ -84,7 +84,7 @@ static void sqlarUncompressFunc( sqlite3_int64 sz; assert( argc==2 ); - sz = sqlite3_value_int(argv[1]); + sz = sqlite3_value_int64(argv[1]); if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); diff --git a/ext/misc/tmstmpvfs.c b/ext/misc/tmstmpvfs.c index 625ffe4fa5..6f1af36f74 100644 --- a/ext/misc/tmstmpvfs.c +++ b/ext/misc/tmstmpvfs.c @@ -484,6 +484,7 @@ static void tmstmpPutU32(u32 v, unsigned char *a){ /* Free a TmstmpLog object */ static void tmstmpLogFree(TmstmpLog *pLog){ if( pLog==0 ) return; + if( pLog->log ) fclose(pLog->log); sqlite3_free(pLog->zLogname); sqlite3_free(pLog); } @@ -502,6 +503,7 @@ static int tmstmpLogFlush(TmstmpFile *p){ } } (void)fwrite(pLog->a, pLog->n, 1, pLog->log); + fflush(pLog->log); pLog->n = 0; return 0; } @@ -619,7 +621,7 @@ static int tmstmpWrite( u32 x = 0; p->iFrame = (iOfst - 32)/(p->pgsz+24)+1; p->pgno = tmstmpGetU32((const u8*)zBuf); - p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16); + p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8); memcpy(&x, ((const u8*)zBuf)+4, 4); p->isCommit = (x!=0); p->iOfst = iOfst; @@ -637,7 +639,7 @@ static int tmstmpWrite( memset(s, 0, TMSTMP_RESERVE); tmstmpPutTS(p, s+2); tmstmpPutU32(p->iFrame, s+8); - tmstmpPutU32(p->pPartner->salt1, s+12); + tmstmpPutU32(p->pPartner->salt1 & 0xffffff, s+12); assert( p->pgsz>0 ); tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0); }else if( p->pPartner==0 ){ @@ -647,7 +649,7 @@ static int tmstmpWrite( tmstmpPutTS(p, s+2); s[12] = 2; assert( p->pgsz>0 ); - tmstmpEvent(p, ELOG_DB_PAGE, 0, (u32)(iOfst/p->pgsz), 0, s+2); + tmstmpEvent(p, ELOG_DB_PAGE, 0, (u32)(iOfst/p->pgsz)+1, 0, s+2); } return pSub->pMethods->xWrite(pSub,zBuf,iAmt,iOfst); } @@ -859,7 +861,7 @@ static int tmstmpOpen( TmstmpLog *pLog; sqlite3_uint64 r1; /* Milliseconds since 1970-01-01 */ sqlite3_uint64 days; /* Days since 1970-01-01 */ - sqlite3_uint64 sod; /* Start of date specified by ms */ + sqlite3_uint64 sod; /* Start of date specified by r1 */ sqlite3_uint64 z; /* Days since 0000-03-01 */ sqlite3_uint64 era; /* 400-year era */ int h; /* hour */ @@ -879,7 +881,9 @@ static int tmstmpOpen( r1 = 0; pLog = sqlite3_malloc64( sizeof(TmstmpLog) ); if( pLog==0 ){ - return SQLITE_NOMEM; + pSubFile->pMethods->xClose(pSubFile); + rc = SQLITE_NOMEM; + goto tmstmp_open_done; } memset(pLog, 0, sizeof(pLog[0])); p->pLog = pLog; @@ -893,7 +897,7 @@ static int tmstmpOpen( m = (sod%3600)/60; s = sod%60; z = days + 719468; - era = z/147097; + era = z/146097; doe = (unsigned)(z - era*146097); yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; y = (int)yoe + era*400; diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 086b058cc5..c4862650b3 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -705,7 +705,12 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; - while( pspec.zTableName) ){ - sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); - }else{ - sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); - } - if( p->spec.bTitles==QRF_Yes ){ - for(i=0; inCol; i++){ - const char *zCName = sqlite3_column_name(p->pStmt, i); - if( qrf_need_quote(zCName) ){ - sqlite3_str_appendf(p->pOut, "%c\"%w\"", - i==0 ? '(' : ',', zCName); - }else{ - sqlite3_str_appendf(p->pOut, "%c%s", - i==0 ? '(' : ',', zCName); + unsigned int mxIns = p->spec.iVersion>=2 ? p->spec.nMultiInsert : 0; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; + } + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } } + sqlite3_str_append(p->pOut, ")", 1); } - sqlite3_str_append(p->pOut, ")", 1); + sqlite3_str_append(p->pOut," VALUES(", 8); + }else{ + sqlite3_str_append(p->pOut,",\n (", 5); } - sqlite3_str_append(p->pOut," VALUES(", 8); for(i=0; inCol; i++){ if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); qrfRenderValue(p, p->pOut, i); } - sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ + sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns = 0; + }else{ + sqlite3_str_append(p->pOut, ")", 1); + } qrfWrite(p); break; } @@ -2630,7 +2647,9 @@ static void qrfOneSimpleRow(Qrf *p){ do{ int nThis, nWide, iNext; qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); - if( cnt ) sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+3,' '); + if( cnt ){ + sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+nSep,' '); + } cnt++; if( cnt>p->mxHeight ){ zVal = "..."; @@ -2693,7 +2712,7 @@ static void qrfInitialize( size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ memset(p, 0, sizeof(*p)); p->pzErr = pzErr; - if( pSpec->iVersion!=1 ){ + if( pSpec->iVersion>2 ){ qrfError(p, SQLITE_ERROR, "unusable sqlite3_qrf_spec.iVersion (%d)", pSpec->iVersion); @@ -2753,6 +2772,7 @@ static void qrfInitialize( if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ p->spec.zTableName = "tab"; } + p->u.nIns = 0; break; } case QRF_STYLE_Line: { @@ -2851,20 +2871,23 @@ static void qrfFinalize(Qrf *p){ switch( p->spec.eStyle ){ case QRF_STYLE_Count: { sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); - qrfWrite(p); break; } case QRF_STYLE_Json: { if( p->nRow>0 ){ sqlite3_str_append(p->pOut, "}]\n", 3); - qrfWrite(p); } break; } case QRF_STYLE_JObject: { if( p->nRow>0 ){ sqlite3_str_append(p->pOut, "}\n", 2); - qrfWrite(p); + } + break; + } + case QRF_STYLE_Insert: { + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); } break; } @@ -2884,15 +2907,14 @@ static void qrfFinalize(Qrf *p){ SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); #endif qrfEqpRender(p, nCycle); - qrfWrite(p); break; } case QRF_STYLE_Eqp: { qrfEqpRender(p, 0); - qrfWrite(p); break; } } + qrfWrite(p); qrfStrErr(p, p->pOut); if( p->spec.pzOutput ){ if( p->spec.pzOutput[0] ){ @@ -2900,7 +2922,7 @@ static void qrfFinalize(Qrf *p){ char *zCombined; sz = strlen(p->spec.pzOutput[0]); n = sqlite3_str_length(p->pOut); - zCombined = sqlite3_realloc(p->spec.pzOutput[0], sz+n+1); + zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); if( zCombined==0 ){ sqlite3_free(p->spec.pzOutput[0]); p->spec.pzOutput[0] = 0; diff --git a/ext/qrf/qrf.h b/ext/qrf/qrf.h index c23ec772f0..fbea33a21f 100644 --- a/ext/qrf/qrf.h +++ b/ext/qrf/qrf.h @@ -56,6 +56,8 @@ struct sqlite3_qrf_spec { void *pRenderArg; /* First argument to the xRender callback */ void *pWriteArg; /* First argument to the xWrite callback */ char **pzOutput; /* Storage location for output string */ + /* Fields below are only available if iVersion>=2 */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ /* Additional fields may be added in the future */ }; diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index e3bcd5fc79..f377d5c30d 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2269,8 +2269,8 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ /* If necessary, grow the pIter->aIdxCol[] array */ if( iIdxCol==nIdxAlloc ){ - RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( - pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( + pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) ); if( aIdxCol==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/repair/README.md b/ext/repair/README.md deleted file mode 100644 index 927ceb7c44..0000000000 --- a/ext/repair/README.md +++ /dev/null @@ -1,16 +0,0 @@ -This folder contains extensions and utility programs intended to analyze -live database files, detect problems, and possibly fix them. - -As SQLite is being used on larger and larger databases, database sizes -are growing into the terabyte range. At that size, hardware malfunctions -and/or cosmic rays will occasionally corrupt a database file. Detecting -problems and fixing errors a terabyte-sized databases can take hours or days, -and it is undesirable to take applications that depend on the databases -off-line for such a long time. -The utilities in the folder are intended to provide mechanisms for -detecting and fixing problems in large databases while those databases -are in active use. - -The utilities and extensions in this folder are experimental and under -active development at the time of this writing (2017-10-12). If and when -they stabilize, this README will be updated to reflect that fact. diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c deleted file mode 100644 index d1d3d54074..0000000000 --- a/ext/repair/checkfreelist.c +++ /dev/null @@ -1,310 +0,0 @@ -/* -** 2017 October 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module exports a single C function: -** -** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); -** -** This function checks the free-list in database zDb (one of "main", -** "temp", etc.) and reports any errors by invoking the sqlite3_log() -** function. It returns SQLITE_OK if successful, or an SQLite error -** code otherwise. It is not an error if the free-list is corrupted but -** no IO or OOM errors occur. -** -** If this file is compiled and loaded as an SQLite loadable extension, -** it adds an SQL function "checkfreelist" to the database handle, to -** be invoked as follows: -** -** SELECT checkfreelist(); -** -** This function performs the same checks as sqlite3_check_freelist(), -** except that it returns all error messages as a single text value, -** separated by newline characters. If the freelist is not corrupted -** in any way, an empty string is returned. -** -** To compile this module for use as an SQLite loadable extension: -** -** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) -# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 -# endif -# if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -# elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -# else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -# endif - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -/* -** Execute a single PRAGMA statement and return the integer value returned -** via output parameter (*pnOut). -** -** The SQL statement passed as the third argument should be a printf-style -** format string containing a single "%s" which will be replace by the -** value passed as the second argument. e.g. -** -** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) -** -** executes "PRAGMA main.page_count" and stores the results in (*pnOut). -*/ -static int sqlGetInteger( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Database name ("main", "temp" etc.) */ - const char *zFmt, /* SQL statement format */ - u32 *pnOut /* OUT: Integer value */ -){ - int rc, rc2; - char *zSql; - sqlite3_stmt *pStmt = 0; - int bOk = 0; - - zSql = sqlite3_mprintf(zFmt, zDb); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnOut = (u32)sqlite3_column_int(pStmt, 0); - bOk = 1; - } - - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; - return rc; -} - -/* -** Argument zFmt must be a printf-style format string and must be -** followed by its required arguments. If argument pzOut is NULL, -** then the results of printf()ing the format string are passed to -** sqlite3_log(). Otherwise, they are appended to the string -** at (*pzOut). -*/ -static int checkFreelistError(char **pzOut, const char *zFmt, ...){ - int rc = SQLITE_OK; - char *zErr = 0; - va_list ap; - - va_start(ap, zFmt); - zErr = sqlite3_vmprintf(zFmt, ap); - if( zErr==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( pzOut ){ - *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); - if( *pzOut==0 ) rc = SQLITE_NOMEM; - }else{ - sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); - } - sqlite3_free(zErr); - } - va_end(ap); - return rc; -} - -static int checkFreelist( - sqlite3 *db, - const char *zDb, - char **pzOut -){ - /* This query returns one row for each page on the free list. Each row has - ** two columns - the page number and page content. */ - const char *zTrunk = - "WITH freelist_trunk(i, d, n) AS (" - "SELECT 1, NULL, sqlite_readint32(data, 32) " - "FROM sqlite_dbpage(:1) WHERE pgno=1 " - "UNION ALL " - "SELECT n, data, sqlite_readint32(data) " - "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " - ")" - "SELECT i, d FROM freelist_trunk WHERE i!=1;"; - - int rc, rc2; /* Return code */ - sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ - u32 nPage = 0; /* Number of pages in db */ - u32 nExpected = 0; /* Expected number of free pages */ - u32 nFree = 0; /* Number of pages on free list */ - - if( zDb==0 ) zDb = "main"; - - if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) - || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) - ){ - return rc; - } - - rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); - while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ - u32 i; - u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); - const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); - u32 iNext = get4byte(&aData[0]); - u32 nLeaf = get4byte(&aData[4]); - - if( nLeaf>((nData/4)-2-6) ){ - rc = checkFreelistError(pzOut, - "leaf count out of range (%d) on trunk page %d", - (int)nLeaf, (int)iTrunk - ); - nLeaf = (nData/4) - 2 - 6; - } - - nFree += 1+nLeaf; - if( iNext>nPage ){ - rc = checkFreelistError(pzOut, - "trunk page %d is out of range", (int)iNext - ); - } - - for(i=0; rc==SQLITE_OK && inPage ){ - rc = checkFreelistError(pzOut, - "leaf page %d is out of range (child %d of trunk page %d)", - (int)iLeaf, (int)i, (int)iTrunk - ); - } - } - } - - if( rc==SQLITE_OK && nFree!=nExpected ){ - rc = checkFreelistError(pzOut, - "free-list count mismatch: actual=%d header=%d", - (int)nFree, (int)nExpected - ); - } - - rc2 = sqlite3_finalize(pTrunk); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ - return checkFreelist(db, zDb, 0); -} - -static void checkfreelist_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const char *zDb; - int rc; - char *zOut = 0; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( nArg==1 ); - zDb = (const char*)sqlite3_value_text(apArg[0]); - rc = checkFreelist(db, zDb, &zOut); - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - - sqlite3_free(zOut); -} - -/* -** An SQL function invoked as follows: -** -** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob -*/ -static void readint_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const u8 *zBlob; - int nBlob; - int iOff = 0; - u32 iRet = 0; - - if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error( - pCtx, "wrong number of arguments to function sqlite_readint32()", -1 - ); - return; - } - if( nArg==2 ){ - iOff = sqlite3_value_int(apArg[1]); - } - - zBlob = sqlite3_value_blob(apArg[0]); - nBlob = sqlite3_value_bytes(apArg[0]); - - if( nBlob>=(iOff+4) ){ - iRet = get4byte(&zBlob[iOff]); - } - - sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); -} - -/* -** Register the SQL functions. -*/ -static int cflRegister(sqlite3 *db){ - int rc = sqlite3_create_function( - db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_create_function( - db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 - ); - return rc; -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkfreelist_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return cflRegister(db); -} diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c deleted file mode 100644 index ed30357e5d..0000000000 --- a/ext/repair/checkindex.c +++ /dev/null @@ -1,929 +0,0 @@ -/* -** 2017 October 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -/* -** Stuff that is available inside the amalgamation, but which we need to -** declare ourselves if this module is compiled separately. -*/ -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -typedef struct CidxTable CidxTable; -typedef struct CidxCursor CidxCursor; - -struct CidxTable { - sqlite3_vtab base; /* Base class. Must be first */ - sqlite3 *db; -}; - -struct CidxCursor { - sqlite3_vtab_cursor base; /* Base class. Must be first */ - sqlite3_int64 iRowid; /* Row number of the output */ - char *zIdxName; /* Copy of the index_name parameter */ - char *zAfterKey; /* Copy of the after_key parameter */ - sqlite3_stmt *pStmt; /* SQL statement that generates the output */ -}; - -typedef struct CidxColumn CidxColumn; -struct CidxColumn { - char *zExpr; /* Text for indexed expression */ - int bDesc; /* True for DESC columns, otherwise false */ - int bKey; /* Part of index, not PK */ -}; - -typedef struct CidxIndex CidxIndex; -struct CidxIndex { - char *zWhere; /* WHERE clause, if any */ - int nCol; /* Elements in aCol[] array */ - CidxColumn aCol[1]; /* Array of indexed columns */ -}; - -static void *cidxMalloc(int *pRc, int n){ - void *pRet = 0; - assert( n!=0 ); - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc(n); - if( pRet ){ - memset(pRet, 0, n); - }else{ - *pRc = SQLITE_NOMEM; - } - } - return pRet; -} - -static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - assert( pCsr->base.pVtab->zErrMsg==0 ); - pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); -} - -/* -** Connect to the incremental_index_check virtual table. -*/ -static int cidxConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int rc = SQLITE_OK; - CidxTable *pRet; - -#define IIC_ERRMSG 0 -#define IIC_CURRENT_KEY 1 -#define IIC_INDEX_NAME 2 -#define IIC_AFTER_KEY 3 -#define IIC_SCANNER_SQL 4 - rc = sqlite3_declare_vtab(db, - "CREATE TABLE xyz(" - " errmsg TEXT," /* Error message or NULL if everything is ok */ - " current_key TEXT," /* SQLite quote() text of key values */ - " index_name HIDDEN," /* IN: name of the index being scanned */ - " after_key HIDDEN," /* IN: Start scanning after this key */ - " scanner_sql HIDDEN" /* debugging info: SQL used for scanner */ - ")" - ); - pRet = cidxMalloc(&rc, sizeof(CidxTable)); - if( pRet ){ - pRet->db = db; - } - - *ppVtab = (sqlite3_vtab*)pRet; - return rc; -} - -/* -** Disconnect from or destroy an incremental_index_check virtual table. -*/ -static int cidxDisconnect(sqlite3_vtab *pVtab){ - CidxTable *pTab = (CidxTable*)pVtab; - sqlite3_free(pTab); - return SQLITE_OK; -} - -/* -** idxNum and idxStr are not used. There are only three possible plans, -** which are all distinguished by the number of parameters. -** -** No parameters: A degenerate plan. The result is zero rows. -** 1 Parameter: Scan all of the index starting with first entry -** 2 parameters: Scan the index starting after the "after_key". -** -** Provide successively smaller costs for each of these plans to encourage -** the query planner to select the one with the most parameters. -*/ -static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ - int iIdxName = -1; - int iAfterKey = -1; - int i; - - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - if( p->usable==0 ) continue; - if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - - if( p->iColumn==IIC_INDEX_NAME ){ - iIdxName = i; - } - if( p->iColumn==IIC_AFTER_KEY ){ - iAfterKey = i; - } - } - - if( iIdxName<0 ){ - pInfo->estimatedCost = 1000000000.0; - }else{ - pInfo->aConstraintUsage[iIdxName].argvIndex = 1; - pInfo->aConstraintUsage[iIdxName].omit = 1; - if( iAfterKey<0 ){ - pInfo->estimatedCost = 1000000.0; - }else{ - pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; - pInfo->aConstraintUsage[iAfterKey].omit = 1; - pInfo->estimatedCost = 1000.0; - } - } - - return SQLITE_OK; -} - -/* -** Open a new btreeinfo cursor. -*/ -static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - CidxCursor *pRet; - int rc = SQLITE_OK; - - pRet = cidxMalloc(&rc, sizeof(CidxCursor)); - - *ppCursor = (sqlite3_vtab_cursor*)pRet; - return rc; -} - -/* -** Close a btreeinfo cursor. -*/ -static int cidxClose(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->zIdxName); - sqlite3_free(pCsr->zAfterKey); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** Move a btreeinfo cursor to the next entry in the file. -*/ -static int cidxNext(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - int rc = sqlite3_step(pCsr->pStmt); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - if( rc!=SQLITE_OK ){ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); - } - }else{ - pCsr->iRowid++; - rc = SQLITE_OK; - } - return rc; -} - -/* We have reached EOF if previous sqlite3_step() returned -** anything other than SQLITE_ROW; -*/ -static int cidxEof(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - return pCsr->pStmt==0; -} - -static char *cidxMprintf(int *pRc, const char *zFmt, ...){ - char *zRet = 0; - va_list ap; - va_start(ap, zFmt); - zRet = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zRet==0 ){ - *pRc = SQLITE_NOMEM; - } - }else{ - sqlite3_free(zRet); - zRet = 0; - } - va_end(ap); - return zRet; -} - -static sqlite3_stmt *cidxPrepare( - int *pRc, CidxCursor *pCsr, const char *zFmt, ... -){ - sqlite3_stmt *pRet = 0; - char *zSql; - va_list ap; /* ... printf arguments */ - va_start(ap, zFmt); - - zSql = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); - if( *pRc!=SQLITE_OK ){ - cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); - } - } - } - sqlite3_free(zSql); - va_end(ap); - - return pRet; -} - -static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ - int rc = sqlite3_finalize(pStmt); - if( *pRc==SQLITE_OK ) *pRc = rc; -} - -char *cidxStrdup(int *pRc, const char *zStr){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - int n = (int)strlen(zStr); - zRet = cidxMalloc(pRc, n+1); - if( zRet ) memcpy(zRet, zStr, n+1); - } - return zRet; -} - -static void cidxFreeIndex(CidxIndex *pIdx){ - if( pIdx ){ - int i; - for(i=0; inCol; i++){ - sqlite3_free(pIdx->aCol[i].zExpr); - } - sqlite3_free(pIdx->zWhere); - sqlite3_free(pIdx); - } -} - -static int cidx_isspace(char c){ - return c==' ' || c=='\t' || c=='\r' || c=='\n'; -} - -static int cidx_isident(char c){ - return c<0 - || (c>='0' && c<='9') || (c>='a' && c<='z') - || (c>='A' && c<='Z') || c=='_'; -} - -#define CIDX_PARSE_EOF 0 -#define CIDX_PARSE_COMMA 1 /* "," */ -#define CIDX_PARSE_OPEN 2 /* "(" */ -#define CIDX_PARSE_CLOSE 3 /* ")" */ - -/* -** Argument zIn points into the start, middle or end of a CREATE INDEX -** statement. If argument pbDoNotTrim is non-NULL, then this function -** scans the input until it finds EOF, a comma (",") or an open or -** close parenthesis character. It then sets (*pzOut) to point to said -** character and returns a CIDX_PARSE_XXX constant as appropriate. The -** parser is smart enough that special characters inside SQL strings -** or comments are not returned for. -** -** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut -** to point to the first character of the string that is not whitespace -** or part of an SQL comment and returns CIDX_PARSE_EOF. -** -** Additionally, if pbDoNotTrim is not NULL and the element immediately -** before (*pzOut) is an SQL comment of the form "-- comment", then -** (*pbDoNotTrim) is set before returning. In all other cases it is -** cleared. -*/ -static int cidxFindNext( - const char *zIn, - const char **pzOut, - int *pbDoNotTrim /* OUT: True if prev is -- comment */ -){ - const char *z = zIn; - - while( 1 ){ - while( cidx_isspace(*z) ) z++; - if( z[0]=='-' && z[1]=='-' ){ - z += 2; - while( z[0]!='\n' ){ - if( z[0]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - while( cidx_isspace(*z) ) z++; - if( pbDoNotTrim ) *pbDoNotTrim = 1; - }else - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]!='*' || z[1]!='/' ){ - if( z[1]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - z += 2; - }else{ - *pzOut = z; - if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF; - switch( *z ){ - case '\0': - return CIDX_PARSE_EOF; - case '(': - return CIDX_PARSE_OPEN; - case ')': - return CIDX_PARSE_CLOSE; - case ',': - return CIDX_PARSE_COMMA; - - case '"': - case '\'': - case '`': { - char q = *z; - z++; - while( *z ){ - if( *z==q ){ - z++; - if( *z!=q ) break; - } - z++; - } - break; - } - - case '[': - while( *z++!=']' ); - break; - - default: - z++; - break; - } - *pbDoNotTrim = 0; - } - } - - assert( 0 ); - return -1; -} - -static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ - const char *z = zSql; - const char *z1; - int e; - int rc = SQLITE_OK; - int nParen = 1; - int bDoNotTrim = 0; - CidxColumn *pCol = pIdx->aCol; - - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e!=CIDX_PARSE_OPEN ) goto parse_error; - z1 = z+1; - z++; - while( nParen>0 ){ - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e==CIDX_PARSE_EOF ) goto parse_error; - if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ - const char *z2 = z; - if( pCol->zExpr ) goto parse_error; - - if( bDoNotTrim==0 ){ - while( cidx_isspace(z[-1]) ) z--; - if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ - z -= 3; - while( cidx_isspace(z[-1]) ) z--; - }else - if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ - z -= 4; - while( cidx_isspace(z[-1]) ) z--; - } - while( cidx_isspace(z1[0]) ) z1++; - } - - pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); - pCol++; - z = z1 = z2+1; - } - if( e==CIDX_PARSE_OPEN ) nParen++; - if( e==CIDX_PARSE_CLOSE ) nParen--; - z++; - } - - /* Search for a WHERE clause */ - cidxFindNext(z, &z, 0); - if( 0==sqlite3_strnicmp(z, "where", 5) ){ - pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]); - }else if( z[0]!='\0' ){ - goto parse_error; - } - - return rc; - - parse_error: - cidxCursorError(pCsr, "Parse error in: %s", zSql); - return SQLITE_ERROR; -} - -static int cidxLookupIndex( - CidxCursor *pCsr, /* Cursor object */ - const char *zIdx, /* Name of index to look up */ - CidxIndex **ppIdx, /* OUT: Description of columns */ - char **pzTab /* OUT: Table name */ -){ - int rc = SQLITE_OK; - char *zTab = 0; - CidxIndex *pIdx = 0; - - sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pInfo = 0; - - /* Find the table for this index. */ - pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name, sql FROM sqlite_schema WHERE name=%Q AND type='index'", - zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ - const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); - zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - nAlloc += 8; - } - p = &pIdx->aCol[iCol++]; - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - if( zSql==0 || p->bKey==0 ){ - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - }else{ - p->zExpr = 0; - } - pIdx->nCol = iCol; - pIdx->zWhere = 0; - } - cidxFinalize(&rc, pInfo); - } - - if( rc==SQLITE_OK && zSql ){ - rc = cidxParseSQL(pCsr, pIdx, zSql); - } - } - - cidxFinalize(&rc, pFindTab); - if( rc==SQLITE_OK && zTab==0 ){ - rc = SQLITE_ERROR; - } - - if( rc!=SQLITE_OK ){ - sqlite3_free(zTab); - cidxFreeIndex(pIdx); - }else{ - *pzTab = zTab; - *ppIdx = pIdx; - } - - return rc; -} - -static int cidxDecodeAfter( - CidxCursor *pCsr, - int nCol, - const char *zAfterKey, - char ***pazAfter -){ - char **azAfter; - int rc = SQLITE_OK; - int nAfterKey = (int)strlen(zAfterKey); - - azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); - if( rc==SQLITE_OK ){ - int i; - char *zCopy = (char*)&azAfter[nCol]; - char *p = zCopy; - memcpy(zCopy, zAfterKey, nAfterKey+1); - for(i=0; i='0' && *p<='9') - || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' - ){ - p++; - } - } - - while( *p==' ' ) p++; - if( *p!=(i==(nCol-1) ? '\0' : ',') ){ - goto parse_error; - } - *p++ = '\0'; - } - } - - *pazAfter = azAfter; - return rc; - - parse_error: - sqlite3_free(azAfter); - *pazAfter = 0; - cidxCursorError(pCsr, "%s", "error parsing after value"); - return SQLITE_ERROR; -} - -static char *cidxWhere( - int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull -){ - char *zRet = 0; - const char *zSep = ""; - int i; - - for(i=0; i"), - azAfter[iGt] - ); - }else{ - zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); - } - - return zRet; -} - -#define CIDX_CLIST_ALL 0 -#define CIDX_CLIST_ORDERBY 1 -#define CIDX_CLIST_CURRENT_KEY 2 -#define CIDX_CLIST_SUBWHERE 3 -#define CIDX_CLIST_SUBEXPR 4 - -/* -** This function returns various strings based on the contents of the -** CidxIndex structure and the eType parameter. -*/ -static char *cidxColumnList( - int *pRc, /* IN/OUT: Error code */ - const char *zIdx, - CidxIndex *pIdx, /* Indexed columns */ - int eType /* True to include ASC/DESC */ -){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {"", " DESC"}; - int i; - const char *zSep = ""; - - for(i=0; inCol; i++){ - CidxColumn *p = &pIdx->aCol[i]; - assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); - switch( eType ){ - - case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); - zSep = ","; - break; - - case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); - zSep = "||','||"; - break; - - case CIDX_CLIST_SUBWHERE: - if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - case CIDX_CLIST_SUBEXPR: - if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - default: - assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); - zSep = ", "; - break; - } - } - } - - return zRet; -} - -/* -** Generate SQL (in memory obtained from sqlite3_malloc()) that will -** continue the index scan for zIdxName starting after zAfterKey. -*/ -int cidxGenerateScanSql( - CidxCursor *pCsr, /* The cursor which needs the new statement */ - const char *zIdxName, /* index to be scanned */ - const char *zAfterKey, /* start after this key, if not NULL */ - char **pzSqlOut /* OUT: Write the generated SQL here */ -){ - int rc; - char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - char *zSrcList = 0; - char **azAfter = 0; - CidxIndex *pIdx = 0; - - *pzSqlOut = 0; - rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); - - zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); - zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); - zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); - zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); - - if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); - } - - if( rc==SQLITE_OK ){ - if( zAfterKey==0 ){ - *pzSqlOut = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " - "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i", - zSubExpr, zTab, zSubWhere, zCurrentKey, - zSrcList, zTab, zIdxName, - (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""), - zOrderBy - ); - }else{ - const char *zSep = ""; - char *zSql; - int i; - - zSql = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", - zSubExpr, zTab, zSubWhere, zCurrentKey - ); - for(i=pIdx->nCol-1; i>=0; i--){ - int j; - if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; - for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, "%z" - "%sSELECT * FROM (" - "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s" - ")", - zSql, zSep, zSrcList, zTab, zIdxName, - pIdx->zWhere ? pIdx->zWhere : "", - pIdx->zWhere ? " AND " : "", - zWhere, zOrderBy - ); - zSep = " UNION ALL "; - if( pIdx->aCol[i].bDesc==0 ) break; - } - } - *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); - } - } - - sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(zSrcList); - cidxFreeIndex(pIdx); - sqlite3_free(azAfter); - return rc; -} - - -/* -** Position a cursor back to the beginning. -*/ -static int cidxFilter( - sqlite3_vtab_cursor *pCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - int rc = SQLITE_OK; - CidxCursor *pCsr = (CidxCursor*)pCursor; - const char *zIdxName = 0; - const char *zAfterKey = 0; - - sqlite3_free(pCsr->zIdxName); - pCsr->zIdxName = 0; - sqlite3_free(pCsr->zAfterKey); - pCsr->zAfterKey = 0; - sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - - if( argc>0 ){ - zIdxName = (const char*)sqlite3_value_text(argv[0]); - if( argc>1 ){ - zAfterKey = (const char*)sqlite3_value_text(argv[1]); - } - } - - if( zIdxName ){ - char *zSql = 0; - pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName); - pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0; - rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql); - if( zSql ){ - pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); - } - } - - if( pCsr->pStmt ){ - assert( rc==SQLITE_OK ); - rc = cidxNext(pCursor); - } - pCsr->iRowid = 1; - return rc; -} - -/* -** Return a column value. -*/ -static int cidxColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, - int iCol -){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL ); - switch( iCol ){ - case IIC_ERRMSG: { - const char *zVal = 0; - if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ - if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ - zVal = "row data mismatch"; - } - }else{ - zVal = "row missing"; - } - sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - break; - } - case IIC_CURRENT_KEY: { - sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); - break; - } - case IIC_INDEX_NAME: { - sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT); - break; - } - case IIC_AFTER_KEY: { - sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT); - break; - } - case IIC_SCANNER_SQL: { - char *zSql = 0; - cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql); - sqlite3_result_text(ctx, zSql, -1, sqlite3_free); - break; - } - } - return SQLITE_OK; -} - -/* Return the ROWID for the sqlite_btreeinfo table */ -static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} - -/* -** Register the virtual table modules with the database handle passed -** as the only argument. -*/ -static int ciInit(sqlite3 *db){ - static sqlite3_module cidx_module = { - 0, /* iVersion */ - 0, /* xCreate */ - cidxConnect, /* xConnect */ - cidxBestIndex, /* xBestIndex */ - cidxDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - cidxOpen, /* xOpen - open a cursor */ - cidxClose, /* xClose - close a cursor */ - cidxFilter, /* xFilter - configure scan constraints */ - cidxNext, /* xNext - advance a cursor */ - cidxEof, /* xEof - check for end of scan */ - cidxColumn, /* xColumn - read data */ - cidxRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ - }; - return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkindex_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return ciInit(db); -} diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in deleted file mode 100644 index 96b15f2713..0000000000 --- a/ext/repair/sqlite3_checker.c.in +++ /dev/null @@ -1,85 +0,0 @@ -/* -** Read an SQLite database file and analyze its space utilization. Generate -** text on standard output. -*/ -#define TCLSH_INIT_PROC sqlite3_checker_init_proc -#define SQLITE_ENABLE_DBPAGE_VTAB 1 -#undef SQLITE_THREADSAFE -#define SQLITE_THREADSAFE 0 -#undef SQLITE_ENABLE_COLUMN_METADATA -#define SQLITE_OMIT_DECLTYPE 1 -#define SQLITE_OMIT_DEPRECATED 1 -#define SQLITE_OMIT_PROGRESS_CALLBACK 1 -#define SQLITE_OMIT_SHARED_CACHE 1 -#define SQLITE_DEFAULT_MEMSTATUS 0 -#define SQLITE_MAX_EXPR_DEPTH 0 -INCLUDE sqlite3.c -INCLUDE $ROOT/src/tclsqlite.c -INCLUDE $ROOT/ext/misc/btreeinfo.c -INCLUDE $ROOT/ext/repair/checkindex.c -INCLUDE $ROOT/ext/repair/checkfreelist.c - -/* -** Decode a pointer to an sqlite3 object. -*/ -int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ - struct SqliteDb *p; - Tcl_CmdInfo cmdInfo; - if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ - p = (struct SqliteDb*)cmdInfo.objClientData; - *ppDb = p->db; - return TCL_OK; - }else{ - *ppDb = 0; - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter -** sqlite3_imposter db main ;# rm all imposters -*/ -static int sqlite3_imposter( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - const char *zSchema; - int iRoot; - const char *zSql; - - if( objc!=3 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - zSchema = Tcl_GetString(objv[2]); - if( objc==3 ){ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); - }else{ - if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; - zSql = Tcl_GetString(objv[4]); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); - sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); - } - return TCL_OK; -} - -#include - -const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3_imposter", - (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); - sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); - return -BEGIN_STRING -INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl -END_STRING -; -} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl deleted file mode 100644 index 2ae6e15b12..0000000000 --- a/ext/repair/sqlite3_checker.tcl +++ /dev/null @@ -1,264 +0,0 @@ -# This TCL script is the main driver script for the sqlite3_checker utility -# program. -# - -# Special case: -# -# sqlite3_checker --test FILENAME ARGS -# -# uses FILENAME in place of this script. -# -if {[lindex $argv 0]=="--test" && [llength $argv]>1} { - set ::argv0 [lindex $argv 1] - set argv [lrange $argv 2 end] - source $argv0 - exit 0 -} - -# Emulate a TCL shell -# -proc tclsh {} { - set line {} - while {![eof stdin]} { - if {$line!=""} { - puts -nonewline "> " - } else { - puts -nonewline "% " - } - flush stdout - append line [gets stdin] - if {[info complete $line]} { - if {[catch {uplevel #0 $line} result]} { - puts stderr "Error: $result" - } elseif {$result!=""} { - puts $result - } - set line {} - } else { - append line \n - } - } -} - -# Do an incremental integrity check of a single index -# -proc check_index {idxname batchsize bTrace} { - set i 0 - set more 1 - set nerr 0 - set pct 00.0 - set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') - WHERE name=$idxname}] - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - if {$bTrace} { - set sql {SELECT errmsg, current_key AS key, - CASE WHEN rowid=1 THEN scanner_sql END AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } else { - set sql {SELECT errmsg, current_key AS key, NULL AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } - while {$more} { - set more 0 - db eval $sql { - set more 1 - if {$errmsg!=""} { - incr nerr - puts "$idxname: key($key): $errmsg" - } elseif {$traceOut!=""} { - puts "$idxname: $traceOut" - } - incr i - - } - set x [format {%.1f} [expr {($i*100.0)/$max}]] - if {$x!=$pct} { - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - set pct $x - } - } - puts "$idxname: $nerr errors out of $i entries" -} - -# Print a usage message on standard error, then quit. -# -proc usage {} { - set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 OPTIONS database-filename" - puts stderr { -Do sanity checking on a live SQLite3 database file specified by the -"database-filename" argument. - -Options: - - --batchsize N Number of rows to check per transaction - - --freelist Perform a freelist check - - --index NAME Run a check of the index NAME - - --summary Print summary information about the database - - --table NAME Run a check of all indexes for table NAME - - --tclsh Run the built-in TCL interpreter (for debugging) - - --trace (Debugging only:) Output trace information on the scan - - --version Show the version number of SQLite -} - exit 1 -} - -set file_to_analyze {} -append argv {} -set bFreelistCheck 0 -set bSummary 0 -set zIndex {} -set zTable {} -set batchsize 1000 -set bAll 1 -set bTrace 0 -set argc [llength $argv] -for {set i 0} {$i<$argc} {incr i} { - set arg [lindex $argv $i] - if {[regexp {^-+tclsh$} $arg]} { - tclsh - exit 0 - } - if {[regexp {^-+version$} $arg]} { - sqlite3 mem :memory: - puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] - mem close - exit 0 - } - if {[regexp {^-+freelist$} $arg]} { - set bFreelistCheck 1 - set bAll 0 - continue - } - if {[regexp {^-+summary$} $arg]} { - set bSummary 1 - set bAll 0 - continue - } - if {[regexp {^-+trace$} $arg]} { - set bTrace 1 - continue - } - if {[regexp {^-+batchsize$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set batchsize [lindex $argv $i] - continue - } - if {[regexp {^-+index$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zIndex [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-+table$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zTable [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-} $arg]} { - puts stderr "Unknown option: $arg" - usage - } - if {$file_to_analyze!=""} { - usage - } else { - set file_to_analyze $arg - } -} -if {$file_to_analyze==""} usage - -# If a TCL script is specified on the command-line, then run that -# script. -# -if {[file extension $file_to_analyze]==".tcl"} { - source $file_to_analyze - exit 0 -} - -set root_filename $file_to_analyze -regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename -if {![file exists $root_filename]} { - puts stderr "No such file: $root_filename" - exit 1 -} -if {![file readable $root_filename]} { - puts stderr "File is not readable: $root_filename" - exit 1 -} - -if {[catch {sqlite3 db $file_to_analyze} res]} { - puts stderr "Cannot open datababase $root_filename: $res" - exit 1 -} - -if {$bFreelistCheck || $bAll} { - puts -nonewline "freelist-check: " - flush stdout - db eval BEGIN - puts [db one {SELECT checkfreelist('main')}] - db eval END -} -if {$bSummary} { - set scale 0 - set pgsz [db one {PRAGMA page_size}] - db eval {SELECT nPage*$pgsz AS sz, name, tbl_name - FROM sqlite_btreeinfo - WHERE type='index' - ORDER BY 1 DESC, name} { - if {$scale==0} { - if {$sz>10000000} { - set scale 1000000.0 - set unit MB - } else { - set scale 1000.0 - set unit KB - } - } - puts [format {%7.1f %s index %s of table %s} \ - [expr {$sz/$scale}] $unit $name $tbl_name] - } -} -if {$zIndex!=""} { - check_index $zIndex $batchsize $bTrace -} -if {$zTable!=""} { - foreach idx [db eval {SELECT name FROM sqlite_master - WHERE type='index' AND rootpage>0 - AND tbl_name=$zTable}] { - check_index $idx $batchsize $bTrace - } -} -if {$bAll} { - set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') - WHERE type='index' AND rootpage>0 - ORDER BY nEntry}] - foreach idx $allidx { - check_index $idx $batchsize $bTrace - } -} diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md deleted file mode 100644 index 8cc954adf5..0000000000 --- a/ext/repair/test/README.md +++ /dev/null @@ -1,13 +0,0 @@ -To run these tests, first build sqlite3_checker: - - -> make sqlite3_checker - - -Then run the "test.tcl" script using: - - -> ./sqlite3_checker --test $path/test.tcl - - -Optionally add the full pathnames of individual *.test modules diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test deleted file mode 100644 index 7e2dd51c37..0000000000 --- a/ext/repair/test/checkfreelist01.test +++ /dev/null @@ -1,92 +0,0 @@ -# 2017-10-11 - -set testprefix checkfreelist - -do_execsql_test 1.0 { - PRAGMA page_size=1024; - CREATE TABLE t1(a, b); -} - -do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.3 { - WITH s(i) AS ( - SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 - ) - INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; - DELETE FROM t1 WHERE rowid%3; - PRAGMA freelist_count; -} {6726} - -do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.5 { - WITH freelist_trunk(i, d, n) AS ( - SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 - UNION ALL - SELECT n, data, sqlite_readint32(data) - FROM freelist_trunk, sqlite_dbpage WHERE pgno=n - ) - SELECT i FROM freelist_trunk WHERE i!=1; -} { - 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 - 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 -} - -do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} - -proc set_int {blob idx newval} { - binary scan $blob I* ints - lset ints $idx $newval - binary format I* $ints -} -db func set_int set_int - -proc get_int {blob idx} { - binary scan $blob I* ints - lindex $ints $idx -} -db func get_int get_int - -do_execsql_test 1.7 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, get_int(data, 1)-1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{free-list count mismatch: actual=6725 header=6726}} - -do_execsql_test 1.8 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.9 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, 0) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.10 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, get_int(data, 1)+1, 0) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 247 of trunk page 5)}} - -do_execsql_test 1.11 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, 249) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf count out of range (249) on trunk page 5}} diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test deleted file mode 100644 index 97973aee71..0000000000 --- a/ext/repair/test/checkindex01.test +++ /dev/null @@ -1,349 +0,0 @@ -# 2017-10-11 -# -set testprefix checkindex - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - CREATE INDEX i1 ON t1(a); - INSERT INTO t1 VALUES('one', 2); - INSERT INTO t1 VALUES('two', 4); - INSERT INTO t1 VALUES('three', 6); - INSERT INTO t1 VALUES('four', 8); - INSERT INTO t1 VALUES('five', 10); - - CREATE INDEX i2 ON t1(a DESC); -} {} - -proc incr_index_check {idx nStep} { - set Q { - SELECT errmsg, current_key FROM incremental_index_check($idx, $after) - LIMIT $nStep - } - - set res [list] - while {1} { - unset -nocomplain current_key - set res1 [db eval $Q] - if {[llength $res1]==0} break - set res [concat $res $res1] - set after [lindex $res end] - } - - return $res -} - -proc do_index_check_test {tn idx res} { - uplevel [list do_execsql_test $tn.1 " - SELECT errmsg, current_key FROM incremental_index_check('$idx'); - " $res] - - uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] -} - - -do_execsql_test 1.2.1 { - SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1'); -} { - 1 1 'five',5 - 2 1 'four',4 - 3 1 'one',1 - 4 1 'three',3 - 5 1 'two',2 -} -do_execsql_test 1.2.2 { - SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql - FROM incremental_index_check('i1') LIMIT 1; -} { - 1 - 'five',5 - i1 - {} - {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i} -} - -do_index_check_test 1.3 i1 { - {} 'five',5 - {} 'four',4 - {} 'one',1 - {} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.4 i2 { - {} 'two',2 - {} 'three',3 - {} 'one',1 - {} 'four',4 - {} 'five',5 -} - -do_test 1.5 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] - sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} - db eval { - UPDATE xt1 SET a='six' WHERE rowid=3; - DELETE FROM xt1 WHERE rowid = 5; - } - sqlite3_imposter db main -} {} - -do_index_check_test 1.6 i1 { - {row missing} 'five',5 - {} 'four',4 - {} 'one',1 - {row data mismatch} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.7 i2 { - {} 'two',2 - {row data mismatch} 'three',3 - {} 'one',1 - {} 'four',4 - {row missing} 'five',5 -} - -#-------------------------------------------------------------------------- -do_execsql_test 2.0 { - - CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); - - INSERT INTO t2 VALUES(1, NULL, 1, 1); - INSERT INTO t2 VALUES(2, 1, NULL, 1); - INSERT INTO t2 VALUES(3, 1, 1, NULL); - - INSERT INTO t2 VALUES(4, 2, 2, 1); - INSERT INTO t2 VALUES(5, 2, 2, 2); - INSERT INTO t2 VALUES(6, 2, 2, 3); - - INSERT INTO t2 VALUES(7, 2, 2, 1); - INSERT INTO t2 VALUES(8, 2, 2, 2); - INSERT INTO t2 VALUES(9, 2, 2, 3); - - CREATE INDEX i3 ON t2(b, c, d); - CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); - CREATE INDEX i5 ON t2(d, c DESC, b); -} {} - -do_index_check_test 2.1 i3 { - {} NULL,1,1,1 - {} 1,NULL,1,2 - {} 1,1,NULL,3 - {} 2,2,1,4 - {} 2,2,1,7 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,3,6 - {} 2,2,3,9 -} - -do_index_check_test 2.2 i4 { - {} 2,2,3,6 - {} 2,2,3,9 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,1,4 - {} 2,2,1,7 - {} 1,1,NULL,3 - {} 1,NULL,1,2 - {} NULL,1,1,1 -} - -do_index_check_test 2.3 i5 { - {} NULL,1,1,3 - {} 1,2,2,4 - {} 1,2,2,7 - {} 1,1,NULL,1 - {} 1,NULL,1,2 - {} 2,2,2,5 - {} 2,2,2,8 - {} 3,2,2,6 - {} 3,2,2,9 -} - -#-------------------------------------------------------------------------- -do_execsql_test 3.0 { - - CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; - CREATE INDEX t3wxy ON t3(w, x, y); - CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); - - INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); - - INSERT INTO t3 VALUES('a', NULL, NULL, 4); - INSERT INTO t3 VALUES('a', NULL, NULL, 5); - INSERT INTO t3 VALUES('a', NULL, NULL, 6); - - INSERT INTO t3 VALUES('a', 'b', NULL, 7); - INSERT INTO t3 VALUES('a', 'b', NULL, 8); - INSERT INTO t3 VALUES('a', 'b', NULL, 9); - -} {} - -do_index_check_test 3.1 t3wxy { - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 -} -do_index_check_test 3.2 t3wxy2 { - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 -} - -#-------------------------------------------------------------------------- -# Test with an index that uses non-default collation sequences. -# -do_execsql_test 4.0 { - CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); - INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); - INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); - INSERT INTO t4 VALUES(3, 'aab', 'ddd'); - INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); - - CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); -} - -do_index_check_test 4.1 t4cc { - {} 'aaa','bbb',1 - {} 'AAA','CCC',2 - {} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -do_test 4.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - - db eval { - UPDATE xt4 SET c1='hello' WHERE rowid=2; - DELETE FROM xt4 WHERE rowid = 3; - } - sqlite3_imposter db main -} {} - -do_index_check_test 4.3 t4cc { - {} 'aaa','bbb',1 - {row data mismatch} 'AAA','CCC',2 - {row missing} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -#-------------------------------------------------------------------------- -# Test an index on an expression. -# -do_execsql_test 5.0 { - CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); - INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); - INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); - INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); - INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); - INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); - - CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); - CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); -} - -do_index_check_test 5.1.1 t5x { - {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 -} - -do_index_check_test 5.1.2 t5y { - {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 -} - -do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { - {} {'{"w":4, "z":4}',4} - {} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -do_test 5.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} - db eval { - UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; - DELETE FROM xt5 WHERE rowid = 4; - } - sqlite3_imposter db main -} {} - -do_index_check_test 5.3.1 t5x { - {row missing} NULL,4 - {row data mismatch} 1,1 - {} 2,2 - {} 3,3 - {} 5,5 -} - -do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { - {row missing} {'{"w":4, "z":4}',4} - {row data mismatch} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -#------------------------------------------------------------------------- -# -do_execsql_test 6.0 { - CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); - CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); - CREATE INDEX t6x2 ON t6(z, -- hello,world, - y); - - CREATE INDEX t6x3 ON t6(z -- hello,world - , y); - - INSERT INTO t6 VALUES(1, 2, 3); - INSERT INTO t6 VALUES(4, 5, 6); -} - -do_index_check_test 6.1 t6x1 { - {} 2,3,1 - {} 5,6,4 -} -do_index_check_test 6.2 t6x2 { - {} 3,2,1 - {} 6,5,4 -} -do_index_check_test 6.2 t6x3 { - {} 3,2,1 - {} 6,5,4 -} - -#------------------------------------------------------------------------- -# -do_execsql_test 7.0 { - CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z); - INSERT INTO t7 VALUES(1, 1, 1); - INSERT INTO t7 VALUES(2, 2, 0); - INSERT INTO t7 VALUES(3, 3, 1); - INSERT INTO t7 VALUES(4, 4, 0); - - CREATE INDEX t7i1 ON t7(y) WHERE z=1; - CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1; - CREATE INDEX t7i3 ON t7(y) WHERE -- yep - z=1; - CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep; -} -do_index_check_test 7.1 t7i1 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.2 t7i2 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.3 t7i3 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.4 t7i4 { - {} 1,1 {} 3,3 -} diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl deleted file mode 100644 index c073bb73c5..0000000000 --- a/ext/repair/test/test.tcl +++ /dev/null @@ -1,67 +0,0 @@ -# Run this script using -# -# sqlite3_checker --test $thisscript $testscripts -# -# The $testscripts argument is optional. If omitted, all *.test files -# in the same directory as $thisscript are run. -# -set NTEST 0 -set NERR 0 - - -# Invoke the do_test procedure to run a single test -# -# The $expected parameter is the expected result. The result is the return -# value from the last TCL command in $cmd. -# -# Normally, $expected must match exactly. But if $expected is of the form -# "/regexp/" then regular expression matching is used. If $expected is -# "~/regexp/" then the regular expression must NOT match. If $expected is -# of the form "#/value-list/" then each term in value-list must be numeric -# and must approximately match the corresponding numeric term in $result. -# Values must match within 10%. Or if the $expected term is A..B then the -# $result term must be in between A and B. -# -proc do_test {name cmd expected} { - if {[info exists ::testprefix]} { - set name "$::testprefix$name" - } - - incr ::NTEST - puts -nonewline $name... - flush stdout - - if {[catch {uplevel #0 "$cmd;\n"} result]} { - puts -nonewline $name... - puts "\nError: $result" - incr ::NERR - } else { - set ok [expr {[string compare $result $expected]==0}] - if {!$ok} { - puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" - incr ::NERR - } else { - puts " Ok" - } - } - flush stdout -} - -# -# do_execsql_test TESTNAME SQL RES -# -proc do_execsql_test {testname sql {result {}}} { - uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] -} - -if {[llength $argv]==0} { - set dir [file dirname $argv0] - set argv [glob -nocomplain $dir/*.test] -} -foreach testfile $argv { - file delete -force test.db - sqlite3 db test.db - source $testfile - catch {db close} -} -puts "$NERR errors out of $NTEST tests" diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 0ae42e7b72..22166a6f9e 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -200,7 +200,7 @@ static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ /* The sqlite3AtoF() routine is much much faster than atof(), if it ** is available */ double r; - (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + (void)sqlite3AtoF((const char*)p->z, &r); *pVal = r; #else *pVal = (GeoCoord)atof((const char*)p->z); diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index b3d29283e5..faebdce78d 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1037,7 +1037,17 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); - assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + if( pRtree->nNodeRef ){ + int i; + assert( pRtree->bCorrupt ); + for(i=0; iaHash[i] ){ + RtreeNode *pNext = pRtree->aHash[i]->pNext; + sqlite3_free(pRtree->aHash[i]); + pRtree->aHash[i] = pNext; + } + } + } sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -2329,7 +2339,7 @@ static int AdjustTree( int iCell; cnt++; - if( NEVER(cnt>100) ){ + if( cnt>100 ){ RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -2687,15 +2697,6 @@ static int SplitNode( rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRight); - pRight = 0; - } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pLeft); - pLeft = 0; - } - splitnode_out: nodeRelease(pRtree, pRight); nodeRelease(pRtree, pLeft); @@ -2880,7 +2881,7 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); - if( ALWAYS(rc==SQLITE_OK) ){ + if( rc==SQLITE_OK ){ if( iHeight==0 ){ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); }else{ diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 4302647609..05375d42bd 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -206,7 +206,7 @@ b.mkdir@ = if [ ! -d $(dir $@) ]; then \ # $1 = logtag, $2 = src file(s). $3 = dest dir b.cp = $(call b.mkdir@); \ echo '$(logtag.$(1)) $(emo.disk) $(2) ==> $3'; \ - cp -p $(2) $(3) || exit + cp -f -p $(2) $(3) || exit # # $(call b.c-pp.shcmd,LOGTAG,src,dest,-Dx=y...) @@ -566,12 +566,14 @@ WASM_CUSTOM_INSTANTIATE = 1 # -D... flags which should be included in all invocations should be # appended to $(b.c-pp.target.flags). # -bin.c-pp = ./c-pp-lite -$(bin.c-pp): c-pp-lite.c $(sqlite3.c) $(MAKEFILE) - $(CC) -O0 -o $@ c-pp-lite.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ +bin.c-pp = ./c-pp +$(bin.c-pp): libcmpp.c $(sqlite3.c) $(MAKEFILE) + $(CC) -O0 -o $@ libcmpp.c $(dir.top)/sqlite3.c -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ - -DSQLITE_TEMP_STORE=3 + -DSQLITE_TEMP_STORE=3 \ + '-DCMPP_DEFAULT_DELIM="//#"' -DCMPP_MAIN -DCMPP_OMIT_D_MODULE \ + -DCMPP_OMIT_D_PIPE DISTCLEAN_FILES += $(bin.c-pp) b.c-pp.target.flags ?= ifeq (1,$(SQLITE_C_IS_SEE)) @@ -911,8 +913,10 @@ ifeq (0,$(wasm-bare-bones)) sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js # Parallel builds can fail if $(sqlite3-license-version.js) is not # created early enough, so make all files in $(sqlite-api.jses) except @@ -1182,13 +1186,13 @@ all: demos ####################################################################### # -# "SOAP" is a static file which is not part of the amalgamation but -# gets copied into the build output folder and into each of the fiddle -# builds. +# "SOAP" is not part of the amalgamation but gets copied into the +# build output folder and into each of the fiddle builds. # sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js -$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js - @$(call b.cp,@,$<,$@) +$(eval $(call b.c-pp.target,soap,\ + $(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\ + $(dir.dout)/sqlite3-opfs-async-proxy.js)) # # Add a dep of $(sqlite3.ext.js) on every individual build's JS file. diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp index 2cdddf1e7c..2b8397a6d7 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp @@ -13,6 +13,7 @@ _sqlite3_bind_parameter_index _sqlite3_bind_parameter_name _sqlite3_bind_pointer _sqlite3_bind_text +_sqlite3_bind_zeroblob _sqlite3_busy_handler _sqlite3_busy_timeout _sqlite3_cancel_auto_extension @@ -225,14 +226,14 @@ _sqlite3session_object_config _sqlite3session_patchset _sqlite3session_patchset_strm _sqlite3session_table_filter -//#endif not bare-bones +//#/if not bare-bones //#if enable-see _sqlite3_key _sqlite3_key_v2 _sqlite3_rekey _sqlite3_rekey_v2 _sqlite3_activate_see -//#endif enable-see +//#/if enable-see //#if fiddle _fiddle_db_arg _fiddle_db_filename @@ -244,4 +245,4 @@ _fiddle_reset_db _fiddle_db_handle _fiddle_db_vfs _fiddle_export_db -//#endif fiddle +//#/if fiddle diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index cac6e4ab4f..b2e760d6a0 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -14,7 +14,7 @@ */ //#if target:es6-module const toExportForESM = -//#endif +//#/if (function(){ //console.warn("this is extern-post-js"); /** @@ -97,7 +97,7 @@ const toExportForESM = //console.warn("sqlite3InitModule() returning E-module.",EmscriptenModule); return EmscriptenModule; } -//#endif +//#/if return s; }).catch((e)=>{ console.error("Exception loading sqlite3 module:",e); @@ -128,10 +128,10 @@ const toExportForESM = } /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ -//#endif // !target:es6-module +//#/if // !target:es6-module return sIM; })(); //#if target:es6-module sqlite3InitModule = toExportForESM; export default sqlite3InitModule; -//#endif +//#/if diff --git a/ext/wasm/api/opfs-common-inline.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js new file mode 100644 index 0000000000..74e911e56d --- /dev/null +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -0,0 +1,191 @@ +//#if 0 +/** + This file is for preprocessor #include into the "opfs" and + "opfs-wl" impls, as well as their async-proxy part. It must be + inlined in those files, as opposed to being a shared copy in the + library, because (A) the async proxy does not load the library and + (B) it references an object which is local to each of those files + but which has a 99% identical structure for each. +*/ +//#/if +//#// vfs.metrics.enable is a refactoring crutch. +//#define vfs.metrics.enable 0 +const initS11n = function(){ + /** + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ +//#if vfs.metrics.enable + ++metrics.s11n.deserialize.count; +//#/if + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); +//#if vfs.metrics.enable + metrics.s11n.deserialize.time += performance.now() - t; +//#/if + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); +//#if vfs.metrics.enable + ++metrics.s11n.serialize.count; +//#/if + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } +//#if vfs.metrics.enable + metrics.s11n.serialize.time += performance.now() - t; +//#/if + }; + +//#if defined opfs-async-proxy + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; +//#/if + + return state.s11n; +//#undef vfs.metrics.enable +}/*initS11n()*/; diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js new file mode 100644 index 0000000000..24ae2632fb --- /dev/null +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -0,0 +1,1301 @@ +//#if not target:node +/* + 2026-03-04 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It + creates a private/internal sqlite3.opfs namespace common to the two + and used (only) by them and the test framework. It is not part of + the public API. The library deletes sqlite3.opfs in its final + bootstrapping steps unless it's specifically told to keep them (for + testing purposes only) using an undocumented and unsupported + mechanism. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + if( sqlite3.config.disable?.vfs?.opfs && + sqlite3.config.disable.vfs['opfs-vfs'] ){ + return; + } + const toss = sqlite3.util.toss, + capi = sqlite3.capi, + util = sqlite3.util, + wasm = sqlite3.wasm; + + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + This is an internal/private namespace intended for use solely by + the OPFS VFSes and test code for them. The library bootstrapping + process removes this object in non-testing contexts. + */ + const opfsUtil = sqlite3.opfs = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + opfsUtil.thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Must be called by the OPFS VFSes immediately after they determine + whether OPFS is available by calling + thisThreadHasOPFS(). Resolves to the OPFS storage root directory + and sets opfsUtil.rootDirectory to that value. + */ + opfsUtil.getRootDir = async function f(){ + return f.promise ??= navigator.storage.getDirectory().then(d=>{ + opfsUtil.rootDirectory = d; + return d; + }).catch(e=>{ + delete f.promise; + throw e; + }); + }; + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = await opfsUtil.getRootDir(); + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + opfsUtil.randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + const dir = await opfsUtil.getRootDir(); + await doDir(dir, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const rd = await opfsUtil.getRootDir(); + const dir = rd, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: await opfsUtil.getRootDir() + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + Impl of opfsUtil.importDb() when it's given a function as its + second argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + /** + Checks for features required for OPFS VFSes and throws with a + descriptive error message if they're not found. This is intended + to be run as part of async VFS installation steps. + */ + opfsUtil.vfsInstallationFeatureCheck = function(vfsName){ + if( !globalThis.SharedArrayBuffer || !globalThis.Atomics ){ + toss("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics.", + "The server must emit the COOP/COEP response headers to enable those.", + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if( 'undefined'===typeof WorkerGlobalScope ){ + toss("The OPFS sqlite3_vfs cannot run in the main thread", + "because it requires Atomics.wait()."); + }else if( !globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || + !navigator?.storage?.getDirectory ){ + toss("Missing required OPFS APIs."); + }else if( 'opfs-wl'===vfsName && !globalThis.Atomics.waitAsync ){ + toss('The',vfsName,'VFS requires Atomics.waitAsync(), which is not available.'); + } + }; + + /** + Must be called by the VFS's main installation routine and passed + the options object that function receives and a reference to that + function itself (we don't need this anymore). + + It throws if OPFS is not available. + + If it returns falsy, it detected that OPFS should be disabled, in + which case the callee should immediately return/resolve to the + sqlite3 object. + + Else it returns a new copy of the options object, fleshed out + with any missing defaults. The caller must: + + - Set up any local state they need. + + - Call opfsUtil.createVfsState(vfsName,opt), where opt is the + object returned by this function. + + - Set up any references they may need to state returned + by the previous step. + + - Call opfvs.bindVfs() + */ + opfsUtil.initOptions = function callee(vfsName, options){ + const urlParams = new URL(globalThis.location.href).searchParams; + if( urlParams.has(vfsName+'-disable') ){ + //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); + return; + } + try{ + opfsUtil.vfsInstallationFeatureCheck(vfsName); + }catch(e){ + return; + } + options = util.nu(options); + options.vfsName = vfsName; + options.verbose ??= urlParams.has('opfs-verbose') + ? +urlParams.get('opfs-verbose') : 1; + options.sanityChecks ??= urlParams.has('opfs-sanity-check'); + + if( !opfsUtil.proxyUri ){ + opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); + } + } + options.proxyUri ??= opfsUtil.proxyUri; + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); + return opfsUtil.options = options; + }; + + /** + Creates, populates, and returns the main state object used by the + "opfs" and "opfs-wl" VFSes, and transfered from those to their + async counterparts. + + The returned object's vfs property holds the fully-populated + capi.sqlite3_vfs instance, tagged with lots of extra state which + the current VFSes need to have exposed to them. + + After setting up any local state needed, the caller must call + theVfs.bindVfs(X,Y), where X is an object containing the + sqlite3_io_methods to override and Y is a callback which gets + triggered if init succeeds, before the final Promise decides + whether or not to reject. + + This object must, when it's passed to the async part, contain + only cloneable or sharable objects. After the worker's "inited" + message arrives, other types of data may be added to it. + */ + opfsUtil.createVfsState = function(){ + const state = util.nu(); + const options = opfsUtil.options; + state.verbose = options.verbose; + + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const vfsName = options.vfsName + || toss("Maintenance required: missing VFS name"); + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level](vfsName+":",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); + const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); + + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString(vfsName); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, opfsIoMethods + /** + Pedantic sidebar: the entries in this array are items to + clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown + order of the wasm engine and the JS one are undefined so + there is no guaranty that the opfsVfs instance would be + available in one environment or the other when + sqlite3_os_end() is called (_if_ it gets called at all in a + wasm build, which is undefined). i.e. addOnDispose() here is + a matter of "correctness", not necessity. It just wouldn't do + to leave the impression that we're blindly leaking memory. + */ + ); + + opfsVfs.metrics = util.nu({ + counters: util.nu(), + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + sqlite3.config.log("Serialization metrics:",opfsVfs.metrics.counters.s11n); + opfsVfs.worker?.postMessage?.({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + const m = opfsVfs.metrics.counters; + for(k in state.opIds){ + r(m[k] = Object.create(null)); + } + let s = m.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = m.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + })/*opfsVfs.metrics*/; + + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 150; + + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). Exceptions related to, e.g., wait/retry + loops in acquiring SyncAccessHandles are not logged. + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + + /** + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. state.opIds holds the SAB slot + IDs of each of those. + */ + state.opIds = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes and clears it. The values written to it + are state.opIds.x[A-Z][a-z]+, defined below.*/ + state.opIds.whichOp = i++; + /* Slot for storing return values. This side listens to that + slot and the async proxy writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to the + state.opIds.whichOp slot. The async-api worker uses + Atomic.wait() on the whichOp slot to figure out which + operation to run next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++ /*currently unused*/; + /** Internal signals which are used only during development and + testing via the dev console. */ + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. It is never written to, only used as a convenient + place to wait-with-timeout for a value which will never be + written, i.e. sleep()ing, before retrying a failed attempt to + acquire a SharedAccessHandle. */ + state.opIds.retry = i++; + state.sabOP = new SharedArrayBuffer( + i * 4/* 4==sizeof int32, noting that Atomics.wait() and + friends can only function on Int32Array views of an + SAB. */); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + for(const k of [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE' + ]){ + state.sq3Codes[k] = + capi[k] ?? toss("Maintenance required: not found:",k); + } + + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under + another instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** + If true, any async routine which must implicitly acquire a + sync access handle (i.e. an OPFS lock), without an active + xLock(), will release that lock at the end of the call which + acquires it. If false, such implicit locks are not released + until the VFS is idle for some brief amount of time, as + defined by state.asyncIdleWaitTime. + + The benefit of enabling this is higher concurrency. The + down-side is much-reduced performance (as much as a 4x + decrease in speedtest1). + */ + defaultUnlockAsap: false + }); + + opfsVfs.metrics.reset()/*must not be called until state.opIds is set up*/; + const metrics = opfsVfs.metrics.counters; + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The 2nd + and subsequent arguments must be the arguments for the async op + (see sqlite3-opfs-async-proxy.c-pp.js). + */ + const opRun = opfsVfs.opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss(opfsVfs.vfsName+": Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync and/or + spuriously wake up. Calling wait() here until it returns + 'not-equal' gets them back in sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its + results in the state.opIds.rc slot of the SAB. It may have + also serialized an exception for us. */ + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = opfsVfs.mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = opfsVfs.mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = opfsVfs.__openFiles = Object.create(null); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = opfsVfs.ioSyncWrappers = util.nu({ + xCheckReservedLock: function(pFile,pOut){ + /** + After consultation with a topic expert: "opfs-wl" will + continue to use the same no-op impl which "opfs" does + because: + + - xCheckReservedLock() is just a hint. If SQLite needs to + lock, it's still going to try to lock. + + - We cannot do this check synchronously in "opfs-wl", + so would need to pass it to the async proxy. That would + make it inordinately expensive considering that it's + just a hint. + */ + wasm.poke(pOut, 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + mTimeStart('xSync'); + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray( + Number(pSrc), Number(pSrc) + n + )); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + })/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = opfsVfs.vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + mTimeEnd(); + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(fh.readOnly){ + wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new capi.sqlite3_file(pFile); + if( zToFree ) fh.sq3File.addOnDispose(zToFree); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + if(pDVfs){ + const dVfs = new capi.sqlite3_vfs(pDVfs); + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + dVfs.dispose(); + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + opfsVfs.vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + const npOut = Number(pOut); + for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + opfsVfs.vfsSyncWrappers.xSleep = function(pVfs,ms){ + mTimeStart('xSleep'); + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + mTimeEnd(); + return 0; + }; + } + +//#define vfs.metrics.enable +//#// import initS11n() +//#include api/opfs-common-inline.c-pp.js +//#undef vfs.metrics.enable + opfsVfs.initS11n = initS11n; + + /** + To be called by the VFS's main installation routine after it has + wired up enough state to provide its overridden io-method impls + (which must be properties of the ioMethods argument). Returns a + Promise which the installation routine must return. callback must + be a function which performs any post-bootstrap touchups, namely + plugging in a sqlite3.oo1 wrapper. It is passed (sqlite3, opfsVfs), + where opfsVfs is the sqlite3_vfs object which was set up by + opfsUtil.createVfsState(). + */ + opfsVfs.bindVfs = function(ioMethods, callback){ + Object.assign(opfsVfs.ioSyncWrappers, ioMethods); + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + try{ + callback(sqlite3, opfsVfs); + }catch(e){ + return promiseReject(e); + } + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + const options = opfsUtil.options; + let proxyUri = options.proxyUri +( + (options.proxyUri.indexOf('?')<0) ? '?' : '&' + )+'vfs='+vfsName; + //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); + const W = opfsVfs.worker = +//#if target:es6-bundler-friendly + (()=>{ + /* _Sigh_... */ + switch(vfsName){ + case 'opfs': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); + case 'opfs-wl': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); + } + })(); +//#elif target:es6-module + new Worker(new URL(proxyUri, import.meta.url)); +//#else + new Worker(proxyUri); +//#/if + let zombieTimer = setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + + const opRun = opfsVfs.opRun; +//#if 0 + /** + Not part of the public API. Only for test/development use. + */ + opfsVfs.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; +//#/if + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new capi.sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(opfsVfs.vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = opfsVfs.ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //sqlite3.config.warn(vfsName,"Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + delete state.vfs; + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + clearTimeout(zombieTimer); + zombieTimer = null; + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + case 'debug': + warn("debug message from worker:",data); + break; + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; + }/*bindVfs()*/; + + return state; + }/*createVfsState()*/; + +}/*sqlite3ApiBootstrap.initializers*/); +//#/if target:node diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 670051bd86..348f80ea0b 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -34,6 +34,7 @@ Module.runSQLite3PostLoadInit = async function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS + - sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 3910cb000b..fbb48f9eac 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -17,9 +17,9 @@ /** This file was preprocessed using: -//#@policy error +//#@ policy error @c-pp::argv@ -//#@policy off +//#@ policy off */ //#if unsupported-build /** @@ -30,9 +30,10 @@ load. It may not work properly. Only builds _directly_ targeting browser environments ("vanilla" JS and ESM modules) are supported and tested. Builds which _indirectly_ target browsers (namely - bundler-friendly builds) are not supported deliverables. + bundler-friendly builds and any node builds) are not supported + deliverables. */ -//#endif +//#/if //#if not target:es6-bundler-friendly (function(Module){ const sIMS = @@ -101,7 +102,7 @@ "result =", theFile ); return theFile; -//#endif target:es6-module +//#/if target:es6-module }.bind(sIMS); //#if Module.instantiateWasm and not wasmfs and not target:node @@ -147,7 +148,7 @@ .then(finalThen) return loadWasm(); }.bind(sIMS); -//#endif Module.instantiateWasm and not wasmfs +//#/if Module.instantiateWasm and not wasmfs })(Module); -//#endif not target:es6-bundler-friendly +//#/if not target:es6-bundler-friendly /* END FILE: api/pre-js.js. */ diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index d268331a35..9c525bd4c7 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -98,6 +98,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_bind_parameter_name", "string", "sqlite3_stmt*", "int"], ["sqlite3_bind_pointer", "int", "sqlite3_stmt*", "int", "*", "string:static", "*"], + /* sqlite_bind_text() is hand-written */ + ["sqlite3_bind_zeroblob", "int", "sqlite3_stmt*", "int", "int"], ["sqlite3_busy_handler","int", [ "sqlite3*", new wasm.xWrap.FuncPtrAdapter({ @@ -120,11 +122,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], -//#define proxy-text-apis=1 +//#define proxy-text-apis 1 //#if not proxy-text-apis /* Search this file for tag:proxy-text-apis to see what this is about. */ ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], -//#endif +//#/if ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_column_value","sqlite3_value*", "sqlite3_stmt*", "int"], ["sqlite3_commit_hook", "void*", [ @@ -324,7 +326,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_value_subtype", "int", "sqlite3_value*"], //#if not proxy-text-apis ["sqlite3_value_text", "string", "sqlite3_value*"], -//#endif +//#/if ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], @@ -500,7 +502,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_activate_see", undefined, "string"] ); } -//#endif enable-see +//#/if enable-see if( wasm.bigIntEnabled && !!wasm.exports.sqlite3_declare_vtab ){ bindingSignatures.int64.push( @@ -1694,7 +1696,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ : null; }; }/*text-return-related bindings*/ -//#endif proxy-text-apis +//#/if proxy-text-apis {/* sqlite3_config() */ /** diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index 9338eef336..13aa427194 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -215,7 +215,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(stmt) stmt.finalize(); } }; -//#endif enable-see +//#/if enable-see /** A proxy for DB class constructors. It must be called with the @@ -301,7 +301,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ //#if enable-see dbCtorApplySEEKey(this,opt); -//#endif +//#/if // Check for per-VFS post-open SQL/callback... const pVfs = capi.sqlite3_js_db_vfs(pDb) || toss3("Internal error: cannot get VFS for new db handle."); @@ -453,7 +453,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ is supplied and the database is encrypted, execution of the post-initialization SQL will fail, causing the constructor to throw. -//#endif enable-see +//#/if enable-see The `filename` and `vfs` arguments may be either JS strings or C-strings allocated via WASM. `flags` is required to be a JS @@ -1141,7 +1141,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return arg.returnVal(); }/*exec()*/, -//#if nope +//#if 0 /** Experimental and untested - do not use. @@ -1265,7 +1265,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } return this; }/*forEachStmt()*/, -//#endif nope +//#/if nope /** Creates a new UDF (User-Defined Function) which is accessible @@ -2426,4 +2426,4 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index c53acee769..e7b775fe51 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -99,6 +99,13 @@ used in WASMFS-capable builds of the library (which the canonical builds do not include). + - `disable` (as of 3.53.0) may be an object with the following + properties: + - `vfs`, an object, may contain a map of VFS names to booleans. + Any mapping to falsy are disabled. The supported names + are: "kvvfs", "opfs", "opfs-sahpool", "opfs-wl". + - Other disabling options may be added in the future. + [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, enabling delayed evaluation. @@ -145,7 +152,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ); return sqlite3ApiBootstrap.sqlite3; } - const config = Object.assign(Object.create(null),{ + const nu = (...obj)=>Object.assign(Object.create(null),...obj); + const config = nu({ exports: undefined, memory: undefined, bigIntEnabled: !!globalThis.BigInt64Array, @@ -162,7 +170,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( certain wasm.xWrap.resultAdapter()s. */ useStdAlloc: false - }, apiConfig || {}); + }, apiConfig); Object.assign(config, { allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', @@ -195,7 +203,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( not documented are installed as 1-to-1 proxies for their C-side counterparts. */ - const capi = Object.create(null); + const capi = nu(); /** Holds state which are specific to the WASM-related infrastructure and glue code. @@ -204,7 +212,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( dynamically after the api object is fully constructed, so not all are documented in this file. */ - const wasm = Object.create(null); + const wasm = nu(); /** Internal helper for SQLite3Error ctor. */ const __rcStr = (rc)=>{ @@ -752,6 +760,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, + nu, assert: function(arg,msg){ if( !arg ){ util.toss("Assertion failed:",msg); @@ -1008,7 +1017,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; }; } - const rc = Object.create(null), ov = [0,0]; + const rc = nu(), ov = [0,0]; let i = 0, k; while((k = capi.sqlite3_compileoption_get(i++))){ f._opt(k,ov); @@ -1016,7 +1025,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } return f._result = rc; }else if(Array.isArray(optName)){ - const rc = Object.create(null); + const rc = nu(); optName.forEach((v)=>{ rc[v] = capi.sqlite3_compileoption_used(v); }); @@ -1067,7 +1076,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( The memory lives in the WASM heap and can be used with routines such as wasm.poke() and wasm.heap8u().slice(). */ - wasm.pstack = Object.assign(Object.create(null),{ + wasm.pstack = nu({ /** Sets the current pstack position to the given pointer. Results are undefined if the passed-in value did not come from @@ -1289,7 +1298,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( // sqlite3__wasm_init_wasmfs() is not available return this.dir = ""; } - }.bind(Object.create(null)); + }.bind(nu()); /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1643,6 +1652,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( case capi.SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE: case capi.SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE: case capi.SQLITE_DBCONFIG_ENABLE_COMMENTS: + case capi.SQLITE_DBCONFIG_FP_DIGITS: if( !this.ip ){ this.ip = wasm.xWrap('sqlite3__wasm_db_config_ip','int', ['sqlite3*', 'int', 'int', '*']); @@ -1664,7 +1674,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( default: return capi.SQLITE_MISUSE; } - }.bind(Object.create(null)); + }.bind(nu()); /** Given a (sqlite3_value*), this function attempts to convert it @@ -1898,7 +1908,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); const pv = wasm.peekPtr(this.ptr); return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); + }.bind(nu()); /** A wrapper around sqlite3_preupdate_new() which fetches the @@ -1938,6 +1948,62 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( 'sqlite3changeset_old'); }/*changeset/preupdate additions*/ + /** + EXPERIMENTAL. For tentative addition in 3.53.0. + + sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry]) + + Calls the given _synchronous_ callback function. If that function + returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error + with a resultCode property of that value then it will suppress + that error and try again, up to the given maximum number of + times. If the callback returns any other value than that, + it is returned. If the maximum number of retries has been + reached, an SQLite3Error with a resultCode value of + sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any + exception other than the aforementioned BUSY exception, it is + propagated. If it throws a BUSY exception on its final attempt, + that is propagated as well. + + If the beforeRetry argument is given, it must be a _synchronous_ + function. It is called immediately before each retry of the + callback (not for the initial call), passed the attempt number + (so it starts with 2, not 1). If it throws, the exception is + handled as described above. Its result value is ignored. + + To effectively retry "forever", pass a negative maxTimes value, + with the caveat that there is no recovery from that unless the + beforeRetry() can figure out when to throw. + + TODO: an async variant of this. + */ + capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){ + for(let n = 1; n <= maxTimes; ++n){ + try{ + if( beforeRetry && n>1 ) beforeRetry(n); + const rc = callback(); + if( capi.SQLITE_BUSY===rc ){ + if( n===maxTimes ){ + throw new SQLite3Error(rc, [ + "sqlite3_js_retry_busy() max retry attempts (", + maxTimes, + ") reached." + ].join('')); + } + continue; + } + return rc; + }catch(e){ + if( npostMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -66,6 +78,13 @@ const installAsyncProxy = function(){ */ const state = Object.create(null); + /* initS11n() is preprocessor-injected so that we have identical + copies in the synchronous and async halves. This side does not + load the SQLite library, so does not have access to that copy. */ +//#define opfs-async-proxy +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-async-proxy + /** verbose: @@ -82,7 +101,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -97,12 +116,13 @@ const installAsyncProxy = function(){ */ const __openFiles = Object.create(null); /** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. + __implicitLocks is a Set of sqlite3_file pointers (integers) + which were "auto-locked". i.e. those for which we necessarily + obtain a sync access handle without an explicit xLock() call + guarding access. Such locks will be released during + `waitLoop()`'s idle time, whereas a sync access handle obtained + via xLock(), or subsequently xLock()'d after auto-acquisition, + will not be released until xUnlock() is called. Maintenance reminder: if we relinquish auto-locks at the end of the operation which acquires them, we pay a massive performance @@ -271,10 +291,11 @@ const installAsyncProxy = function(){ In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to some fixed number of - times. If acquisition still fails at that point it will give up - and propagate the exception. Client-level code will see that as - an I/O error. + will wait briefly and try again, up to `maxTries` of times. If + acquisition still fails at that point it will give up and + propagate the exception. Client-level code will see that either + as an I/O error or SQLITE_BUSY, depending on the exception and + the context. 2024-06-12: there is a rare race condition here which has been reported a single time: @@ -289,13 +310,31 @@ const installAsyncProxy = function(){ there's another race condition there). That's easy to say but creating a viable test for that condition has proven challenging so far. + + Interface quirk: if fh.xLock is falsy and the handle is acquired + then fh.fid is added to __implicitLocks(). If fh.xLock is truthy, + it is not added as an implicit lock. i.e. xLock() impls must set + fh.xLock immediately _before_ calling this and must arrange to + restore it to its previous value if this function throws. + + 2026-03-06: + + - baseWaitTime is the number of milliseconds to wait for the + first retry, increasing by one factor for each retry. It defaults + to (state.asyncIdleWaitTime*2). + + - maxTries is the number of attempt to make, each one spaced out + by one additional factor of the baseWaitTime (e.g. 300, then 600, + then 900, the 1200...). This MUST be an integer >0. + + Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName)=>{ + const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, - msBase = state.asyncIdleWaitTime * 2; + const msBase = baseWaitTime ?? (state.asyncIdleWaitTime * 2); + maxTries ??= 6; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -329,6 +368,9 @@ const installAsyncProxy = function(){ /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. + + The opName is only used for logging and debugging - all result + codes are expected on the same state.sabOPView slot. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); @@ -458,24 +500,12 @@ const installAsyncProxy = function(){ await releaseImplicitLock(fh); storeAndNotify('xFileSize', rc); }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - } - storeAndNotify('xLock',rc); - }, + /** + The first argument is semantically invalid here - it's an + address in the synchronous side's heap. We can do nothing with + it here except use it as a unique-per-file identifier. + i.e. a lookup key. + */ xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ @@ -533,7 +563,7 @@ const installAsyncProxy = function(){ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - error("xRead() failed",e,fh); + //error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } @@ -560,29 +590,13 @@ const installAsyncProxy = function(){ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ - error("xTruncate():",e,fh); + //error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } await releaseImplicitLock(fh); storeAndNotify('xTruncate',rc); }, - xUnlock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - let rc = 0; - const fh = __openFiles[fid]; - if( fh.syncHandle - && state.sq3Codes.SQLITE_LOCK_NONE===lockType - /* Note that we do not differentiate between lock types in - this VFS. We're either locked or unlocked. */ ){ - try { await closeSyncHandle(fh) } - catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; - } - } - storeAndNotify('xUnlock',rc); - }, xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ let rc; const fh = __openFiles[fid]; @@ -594,7 +608,7 @@ const installAsyncProxy = function(){ {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ - error("xWrite():",e,fh); + //error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } @@ -603,152 +617,284 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; - const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". + if( isWebLocker ){ + /* We require separate xLock() and xUnlock() implementations for the + original and Web Lock implementations. The ones in this block + are for the WebLock impl. + + The Golden Rule for this impl is: if we have a web lock, we + must also hold the SAH. When "upgrading" an implicit lock to a + requested (explicit) lock, we must remove the SAH from the + __implicitLocks set. When we unlock, we release both the web + lock and the SAH. That invariant must be kept intact or race + conditions on SAHs will ensue. */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); + /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ + const __activeWebLocks = Object.create(null); + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/, + isFromUnlock/*only if called from this.xUnlock()*/){ + const whichOp = isFromUnlock ? 'xUnlock' : 'xLock'; + const fh = __openFiles[fid]; + //error("xLock()",fid, lockType, isFromUnlock, fh); + const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED) + ? 'exclusive' : 'shared'; + const existing = __activeWebLocks[fid]; + if( existing ){ + if( existing.mode === requestedMode + || (existing.mode === 'exclusive' + && requestedMode === 'shared') ) { + fh.xLock = lockType; + storeAndNotify(whichOp, 0); + /* Don't do this: existing.mode = requestedMode; + + Paraphrased from advice given by a consulting developer: + + If you hold an exclusive lock and SQLite requests shared, + you should keep exiting.mode as exclusive in because the + underlying Web Lock is still exclusive. Changing it to + shared would trick xLock into thinking it needs to + perform a release/re-acquire dance if an exclusive is + later requested. + */ + return 0 /* Already held at required or higher level */; } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; + /* + Upgrade path: we must release shared and acquire exclusive. + This transition is NOT atomic in Web Locks API. + + It _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that until + we let it go. But we can't do that without eventually + leading to deadly embrace situations, so we don't do that. + (That's not a hypothetical, it has happened.) + */ + await closeSyncHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + } + + const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; + const oldLockType = fh.xLock; + return new Promise((resolveWaitLoop) => { + //error("xLock() initial promise entered..."); + navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { + //error("xLock() Web Lock entered.", fh); + __implicitLocks.delete(fid); + let rc = 0; + try{ + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; + await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); + }catch(e){ + fh.xLock = oldLockType; + state.s11n.storeException(1, e); + rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_BUSY); } - rc.push(v); - } + const releasePromise = rc + ? undefined + : new Promise((resolveRelease) => { + __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; + }); + storeAndNotify(whichOp, rc) /* unblock the C side */; + resolveWaitLoop(0) /* unblock waitLoop() */; + await releasePromise /* hold the lock until xUnlock */; + }); + }); + }; + + /** Internal helper for the opfs-wl xUnlock() */ + const wlCloseHandle = async(fh)=>{ + let rc = 0; + try{ + /* For the record, we've never once seen closeSyncHandle() + throw, nor should it because destructors do not throw. */ + await closeSyncHandle(fh); + }catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); return rc; }; - state.s11n.serialize = function(...args){ - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + const existing = __activeWebLocks[fid]; + if( !existing ){ + const rc = await wlCloseHandle(fh); + storeAndNotify('xUnlock', rc); + return rc; + } + //error("xUnlock()",fid, lockType, fh); + let rc = 0; + if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ + /* SQLite usually unlocks all the way to NONE */ + rc = await wlCloseHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + fh.xLock = lockType; + }else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED + && existing.mode === 'exclusive' ){ + /* downgrade Exclusive -> Shared */ + rc = await wlCloseHandle(fh); + if( 0===rc ){ + fh.xLock = lockType; + existing.resolveRelease(); + delete __activeWebLocks[fid]; + return vfsAsyncImpls.xLock(fid, lockType, true); } - //log("serialize() result:",viewU8.slice(0,offset)); }else{ - viewU8[0] = 0; + /* ??? */ + error("xUnlock() unhandled condition", fh); } + storeAndNotify('xUnlock', rc); + return 0; + } + + }else{ + /* Original/"legacy" xLock() and xUnlock() */ + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } + } + storeAndNotify('xLock',rc); }; - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + let rc = 0; + const fh = __openFiles[fid]; + if( fh.syncHandle + && state.sq3Codes.SQLITE_LOCK_NONE===lockType + /* Note that we do not differentiate between lock types in + this VFS. We're either locked or unlocked. */ ){ + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - }) - : ()=>{}; + } + storeAndNotify('xUnlock',rc); + } - return state.s11n; - }/*initS11n()*/; + }/*xLock() and xUnlock() impls*/ const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; + if( !f.inited ){ + f.inited = true; + f.opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + f.opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } } + const opIds = state.opIds; + const opView = state.sabOPView; + const slotWhichOp = opIds.whichOp; + const idleWaitTime = state.asyncIdleWaitTime; + const hasWaitAsync = !!Atomics.waitAsync; +//#if 0 + error("waitLoop init: isWebLocker",isWebLocker, + "idleWaitTime",idleWaitTime, + "hasWaitAsync",hasWaitAsync); +//#/if while(!flagAsyncShutdown){ try { - if('not-equal'!==Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime - )){ - /* Maintenance note: we compare against 'not-equal' because - - https://github.com/tomayac/sqlite-wasm/issues/12 - - is reporting that this occasionally, under high loads, - returns 'ok', which leads to the whichOp being 0 (which - isn't a valid operation ID and leads to an exception, - along with a corresponding ugly console log - message). Unfortunately, the conditions for that cannot - be reliably reproduced. The only place in our code which - writes a 0 to the state.opIds.whichOp SharedArrayBuffer - index is a few lines down from here, and that instance - is required in order for clear communication between - the sync half of this proxy and this half. + let opId; + if( hasWaitAsync ){ + opId = Atomics.load(opView, slotWhichOp); + if( 0===opId ){ + const rv = Atomics.waitAsync(opView, slotWhichOp, 0, + idleWaitTime); + if( rv.async ) await rv.value; + await releaseImplicitLocks(); + continue; + } + }else{ + /** + For browsers without Atomics.waitAsync(), we require + the legacy implementation. Browser versions where + waitAsync() arrived: + + Chrome: 90 (2021-04-13) + Firefox: 145 (2025-11-11) + Safari: 16.4 (2023-03-27) + + The "opfs" VFS was not born until Chrome was somewhere in + the v104-108 range (Summer/Autumn 2022) and did not work + with Safari < v17 (2023-09-18) due to a WebKit bug which + restricted OPFS access from sub-Workers. + + The waitAsync() counterpart of this block can be used by + both "opfs" and "opfs-wl", whereas this block can only be + used by "opfs". Performance comparisons between the two + in high-contention tests have been indecisive. */ - await releaseImplicitLocks(); - continue; + if('not-equal'!==Atomics.wait( + state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime + )){ + /* Maintenance note: we compare against 'not-equal' because + + https://github.com/tomayac/sqlite-wasm/issues/12 + + is reporting that this occasionally, under high loads, + returns 'ok', which leads to the whichOp being 0 (which + isn't a valid operation ID and leads to an exception, + along with a corresponding ugly console log + message). Unfortunately, the conditions for that cannot + be reliably reproduced. The only place in our code which + writes a 0 to the state.opIds.whichOp SharedArrayBuffer + index is a few lines down from here, and that instance + is required in order for clear communication between + the sync half of this proxy and this half. + + Much later (2026-03-07): that phenomenon is apparently + called a spurious wakeup. + */ + await releaseImplicitLocks(); + continue; + } + opId = Atomics.load(state.sabOPView, slotWhichOp); } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + Atomics.store(opView, slotWhichOp, 0); + const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */ ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); + //error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args); +//#if 0 + if( isWebLocker && (opId==opIds.xLock || opIds==opIds.xUnlock) ){ + /* An expert suggests that this introduces a race condition, + but my eyes aren't seeing it. The hope was that this + would improve the lock speed a tick, but it does not + appear to. */ + hnd(...args); + continue; + } +//#/if + await hnd(...args); }catch(e){ - error('in waitLoop():',e); + error('in waitLoop():', e); } } }; @@ -756,6 +902,7 @@ const installAsyncProxy = function(){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; globalThis.onmessage = function({data}){ + //log(globalThis.location.href,"onmessage()",data); switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -771,6 +918,7 @@ const installAsyncProxy = function(){ } }); initS11n(); + //warn("verbosity =",opt.verbose, state.verbose); log("init state",state); wPost('opfs-async-inited'); waitLoop(); @@ -782,22 +930,27 @@ const installAsyncProxy = function(){ flagAsyncShutdown = false; waitLoop(); } - break; + break; } }; wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!globalThis.SharedArrayBuffer){ +if(globalThis.window === globalThis){ + wPost('opfs-unavailable', + "This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); }else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); +}else if(isWebLocker && !globalThis.Atomics.waitAsync){ + wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName); }else if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js index a78c0548b7..0db303bc46 100644 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -25,9 +25,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ delete sqlite3.capi.KVVfsFile; } //#else -//#@policy error +//#@ policy error //#savepoint begin -//#define kvvfs-v2-added-in=3.52.0 +//#define kvvfs-v2-added-in "3.52.0" + /** kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates storage of its pages and metadata to a key-value store. @@ -85,6 +86,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ backups. */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( sqlite3.config.disable?.vfs?.kvvfs ){ + return; + } 'use strict'; const capi = sqlite3.capi, sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, @@ -209,17 +213,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and recreating it whenever a property index might be invalidated. */ class KVVfsStorage { - #map; - #keys; - #getKeys(){return this.#keys ??= Object.keys(this.#map);} + #map = Object.create(null); + #keys = null; + #size = 0; constructor(){ this.clear(); } + #getKeys(){ + return this.#keys ??= Object.keys(this.#map); + } + key(n){ - const k = this.#getKeys(); - return n= this.#size) return null; + return this.#getKeys()[n]; } getItem(k){ @@ -227,14 +235,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } setItem(k,v){ - if( !hop(this.#map, k) ){ + if( !(k in this.#map) ){ + ++this.#size; this.#keys = null; } this.#map[k] = ''+v; } removeItem(k){ - if( delete this.#map[k] ){ + if( k in this.#map ){ + delete this.#map[k]; + --this.#size; this.#keys = null; } } @@ -242,10 +253,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ clear(){ this.#map = Object.create(null); this.#keys = null; + this.#size = 0; } get length() { - return this.#getKeys().length; + return this.#size; } }/*KVVfsStorage*/; @@ -569,7 +581,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ? cache.storagePool[zClass] : cache.storagePool[wasm.cstrToJs(zClass)]; -//#if nope +//#if 0 // fileForDb() works but we don't have a current need for it. /** Expects an (sqlite3*). Uses sqlite3_file_control() to extract its @@ -612,7 +624,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ throw e; } }; -//#endif nope +//#/if const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey; /** @@ -956,7 +968,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return 0; } -//#if nope +//#if 0 // these impls work but there's currently no pressing need _not_ use // the native impls. xCurrentTime: function(pVfs,pOut){ @@ -968,7 +980,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now()); return 0; } -//#endif +//#/if }/*.vfs*/, /** @@ -1064,7 +1076,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }, -//#if not nope +//#if 0 // We override xRead/xWrite only for logging/debugging. They // should otherwise be disabled (it's faster that way). xRead: function(pFile,pTgt,n,iOff64){ @@ -1095,9 +1107,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return cache.setError(e); } }, -//#endif nope +//#/if -//#if nope +//#if 0 xTruncate: function(pFile,i64){}, xFileSize: function(pFile,pi64Out){}, xLock: function(pFile,iLock){}, @@ -1105,7 +1117,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xCheckReservedLock: function(pFile,piOut){}, xSectorSize: function(pFile){}, xDeviceCharacteristics: function(pFile){} -//#endif +//#/if }/*.ioDb*/, ioJrnl:{ @@ -1113,7 +1125,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ are copied as-is from the ioDb objects. Others are specific to journal files. */ xClose: true, -//#if nope +//#if 0 xRead: function(pFile,pTgt,n,iOff64){}, xWrite: function(pFile,pSrc,n,iOff64){}, xTruncate: function(pFile,i64){}, @@ -1125,13 +1137,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xCheckReservedLock: true, xSectorSize: true, xDeviceCharacteristics: true -//#endif +//#/if }/*.ioJrnl*/ }/*methodOverrides*/; +//#if 0 debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, kvvfsMethods, capi.sqlite3_file.structInfo, KVVfsFile.structInfo); +//#/if + try { util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); @@ -1220,7 +1235,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ limitation which has since been overcome, but removal of JsStorageDb.prototype.clearStorage() would be a backwards compatibility break, so this function permits wiping the storage for those two - cases even if they are opened. Use with case. + cases even if they are opened. Use with care. */ const sqlite3_js_kvvfs_clear = function callee(which){ if( ''===which ){ @@ -1829,7 +1844,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return jdb.storageSize(this.affirmOpen().dbFilename(), true); }; }/*sqlite3.oo1.JsStorageDb*/ -//#endif not omit-oo1 +//#/if not omit-oo1 if( sqlite3.__isUnderTest && sqlite3.vtab ){ /** @@ -1993,7 +2008,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/* virtual table */ -//#if nope +//#if 0 /** The idea here is a simpler wrapper for listening to kvvfs changes. Clients would override its onXyz() event methods @@ -2080,8 +2095,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ async onOpen(count){} async onClose(count){} }/*KvvfsListener*/; -//#endif nope +//#/if nope })/*globalThis.sqlite3ApiBootstrap.initializers*/; //#savepoint rollback -//#endif not omit-kvvfs +//#/if not omit-kvvfs diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index c1fee5e1ab..2990fb1470 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -55,6 +55,10 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; + if( sqlite3.config.disable?.vfs?.['opfs-sahpool'] ){ + return; + } + const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */; @@ -1467,4 +1471,4 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The OPFS SAH Pool VFS parts are elided from builds targeting node.js. */ -//#endif target:node +//#/if target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js new file mode 100644 index 0000000000..30b24869d6 --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -0,0 +1,133 @@ +//#if not target:node +/* + 2026-02-20 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is a reimplementation of the "opfs" VFS (as distinct from + "opfs-sahpool") which uses WebLocks for locking instead of a bespoke + custom Atomics.wait()/notify() protocol. This file holds the + "synchronous half" of the VFS, whereas it shares the "asynchronous + half" of the "opfs" VFS. + + This file is intended to be appended to the main sqlite3 JS + deliverable somewhere after sqlite3-api-oo1.js. + + TODOs (2026-03-03): + + - For purposes of tester1.js we need to figure out which of these + VFSes will install the (internal-use-only) sqlite3.opfs utility code + namespace. We need that in order to clean up OPFS files during test + runs. Alternately, move those into their own + sqlite3ApiBootstrap.initializers entry which precedes both of the + VFSes, so they'll have access to it during bootstrapping. The + sqlite3.opfs namespace is removed at the end of bootstrapping unless + the library is told to run in testing mode (which is not a + documented feature). +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.['opfs-wl'] ){ + return; + } + const util = sqlite3.util, + toss = sqlite3.util.toss; + const opfsUtil = sqlite3.opfs; + const vfsName = 'opfs-wl'; +/** + installOpfsWlVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializers or an equivalent mechanism. + + This VFS is essentially identical to the "opfs" VFS but uses + WebLocks for its xLock() and xUnlock() implementations. + + Quirks specific to this VFS: + + - The (officially undocumented) 'opfs-wl-disable' URL + argument will disable OPFS, making this function a no-op. + + Aside from locking differences in the VFSes, this function + otherwise behaves the same as + sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). +*/ +const installOpfsWlVfs = async function(options){ + options = opfsUtil.initOptions(vfsName,options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug(vfsName+":",...args), + warn = (...args)=>sqlite3.config.warn(vfsName+":",...args), + __openFiles = opfsVfs.__openFiles; + + //debug("state",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + //debug("xLock()..."); + const f = __openFiles[pFile]; + const rc = opRun('xLock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + const rc = opRun('xUnlock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + } + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ + if(sqlite3.oo1){ + const OpfsWlDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = vfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsWlDb = OpfsWlDb; + OpfsWlDb.importDb = opfsUtil.importDb; + /* The "opfs" VFS variant adds a + oo1.DB.dbCtorHelper.setVfsPostOpenCallback() callback to set + a high busy_timeout. That was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is retained + against the risk of introducing regressions if it's removed. + This variant does not repeat that mistake. + */ + } + })/*bindVfs()*/; +}/*installOpfsWlVfs()*/; +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install the",vfsName,"sqlite3_vfs:",e); + }); +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); +//#/if target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ffa90ed068..8edfab4dab 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -20,7 +20,12 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.opfs ){ + return; + } + const util = sqlite3.util, + opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); + /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via @@ -58,7 +63,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The argument may optionally be a plain object with the following configuration options: - - proxyUri: name of the async proxy JS file. + - proxyUri: name of the async proxy JS file or a synchronous function + which, when called, returns such a name. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables logging of errors. 2 enables logging of warnings and errors. 3 @@ -70,1391 +76,105 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Promise resolves. This is only intended for testing and development of the VFS, not client-side use. + Additionaly, the (officially undocumented) 'opfs-disable' URL + argument will disable OPFS, making this function a no-op. + On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. Success does not necessarily mean that it installs the VFS, + as there are legitimate non-error reasons for OPFS not to be + available. */ -const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); - } - const urlParams = new URL(globalThis.location.href).searchParams; - if(urlParams.has('opfs-disable')){ - //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); - return Promise.resolve(sqlite3); - } - if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; - } - if(undefined===options.sanityChecks){ - options.sanityChecks = urlParams.has('opfs-sanity-check'); - } - if(undefined===options.proxyUri){ - options.proxyUri = callee.defaultProxyUri; - } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - - if('function' === typeof options.proxyUri){ - options.proxyUri = options.proxyUri(); - } - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. - - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); - - /** - Returns true if _this_ thread has access to the OPFS APIs. - */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; - - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - sqlite3.config.log(globalThis.location.href, - "metrics for",globalThis.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - sqlite3.config.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*metrics*/; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - const W = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - asyncIdleWaitTime is how long (ms) to wait, in the async proxy, - for each Atomics.wait() when waiting on inbound VFS API calls. - We need to wake up periodically to give the thread a chance to - do other things. If this is too high (e.g. 500ms) then even two - workers/tabs can easily run into locking errors. Some multiple - of this value is also used for determining how long to wait on - lock contention to free up. - */ - state.asyncIdleWaitTime = 150; - - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging. - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. We also - store exceptions there, so it needs to be long enough to hold - a reasonably long exception string. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - const metrics = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_BUSY', - 'SQLITE_CANTOPEN', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCKED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - state.opfsFlags = Object.assign(Object.create(null),{ - /** - Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" - enables this. See defaultUnlockAsap, below. - */ - OPFS_UNLOCK_ASAP: 0x01, - /** - Flag for use with xOpen(). URI flag "delete-before-open=1" - tells the VFS to delete the db file before attempting to open - it. This can be used, e.g., to replace a db which has been - corrupted (without forcing us to expose a delete/unlink() - function in the public API). - - Failure to unlink the file is ignored but may lead to - downstream errors. An unlink can fail if, e.g., another tab - has the handle open. - - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. - */ - OPFS_UNLINK_BEFORE_OPEN: 0x02, - /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). - */ - defaultUnlockAsap: false - }); - - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ - /* - The reason for this loop is buried in the details of a long - discussion at: - - https://github.com/sqlite/sqlite-wasm/issues/12 - - Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. - */ - } - /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); +const installOpfsVfs = async function(options){ + options = opfsUtil.initOptions('opfs',options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug("opfs:",...args), + warn = (...args)=>sqlite3.config.warn("opfs:",...args), + __openFiles = opfsVfs.__openFiles; + + //debug("options:",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + ++metrics.xLock.count; + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { + f.lockType = lockType; + }else{ + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; } + mTimeEnd(); return rc; - }; - - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile is open. - - Update 2024-06-12: based on forum discussions, this - function now always sets pOut to 0 (false): - - https://sqlite.org/forum/forumpost/a2f573b00cda1372 - */ - wasm.poke(pOut, 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - /*mTimeStart('xFileControl'); - mTimeEnd();*/ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - let rc = opRun('xFileSize', pFile); - if(0==rc){ - try { - const sz = state.s11n.deserialize()[0]; - wasm.poke(pSz64, sz, 'i64'); - }catch(e){ - error("Unexpected error reading xFileSize() result:",e); - rc = state.sq3Codes.SQLITE_IOERR; - } - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile,lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( !f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - mTimeStart('xSync'); - ++metrics.xSync.count; - const rc = opRun('xSync', pFile, flags); - mTimeEnd(); - return rc; - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + ++metrics.xUnlock.count; + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } - if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray( - Number(pSrc), Number(pSrc) + n - )); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; + rc = opRun('xUnlock', pFile, lockType); } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstrToJs(zName)); - wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); - mTimeEnd(); - return rc; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); - }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; + if( true ){ + /* 2026-03-06: this was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is + retained against the risk of introducing regressions if + it's removed. */ + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + ); + } + }/*extend sqlite3.oo1*/ + })/*bindVfs()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - try{ - let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ - installOpfsVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); - } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); - }); - }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); - return Promise.reject(e); - } + return installOpfsVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install 'opfs' sqlite3_vfs:",e); + }) }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ -//#endif target:node +//#/if target:node diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index dbfcdb7047..0c5f4f8ea5 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -571,6 +571,7 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ DefInt(SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE); DefInt(SQLITE_DBCONFIG_ENABLE_COMMENTS); DefInt(SQLITE_DBCONFIG_MAX); + DefInt(SQLITE_DBCONFIG_FP_DIGITS); } _DefGroup; DefGroup(dbStatus){ @@ -698,6 +699,8 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ DefInt(SQLITE_MAX_TRIGGER_DEPTH); DefInt(SQLITE_LIMIT_WORKER_THREADS); DefInt(SQLITE_MAX_WORKER_THREADS); + DefInt(SQLITE_LIMIT_PARSER_DEPTH); + DefInt(SQLITE_MAX_PARSER_DEPTH); } _DefGroup; DefGroup(openFlags) { @@ -731,6 +734,7 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ DefInt(SQLITE_PREPARE_PERSISTENT); DefInt(SQLITE_PREPARE_NORMALIZE); DefInt(SQLITE_PREPARE_NO_VTAB); + DefInt(SQLITE_PREPARE_FROM_DDL); } _DefGroup; DefGroup(resultCodes) { @@ -1640,6 +1644,7 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_ip, case SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE: case SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE: case SQLITE_DBCONFIG_ENABLE_COMMENTS: + case SQLITE_DBCONFIG_FP_DIGITS: return sqlite3_db_config(pDb, op, arg1, pArg2); default: return SQLITE_MISUSE; } diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index b282c5e6e1..bcbf3fa9f8 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -24,7 +24,7 @@ */ //#if not defined target:es6-module 'use strict'; -//#endif +//#/if /** Configures an sqlite3 Worker API #1 Worker such that it can be manipulated via a Promise-based interface and returns a factory @@ -278,13 +278,13 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { } } return new Worker(theJs + globalThis.location.search); -//#endif +//#/if } //#if not target:es6-module .bind({ currentScript: globalThis?.document?.currentScript }) -//#endif +//#/if , onerror: (...args)=>console.error('sqlite3Worker1Promiser():',...args) }/*defaultConfig*/; @@ -349,7 +349,7 @@ globalThis.sqlite3Worker1Promiser.v2.defaultConfig = */ export default sqlite3Worker1Promiser.v2; delete globalThis.sqlite3Worker1Promiser; -//#endif /* target:es6-module */ +//#/if /* target:es6-module */ //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index db27c8fc0e..046243baa5 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -49,8 +49,8 @@ import sqlite3InitModule from './sqlite3.mjs'; //console.warn("worker1 theJs =",theJs); importScripts(theJs); } -//#endif +//#/if sqlite3InitModule().then(sqlite3 => sqlite3.initWorker1API()); //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/c-pp-lite.c b/ext/wasm/c-pp-lite.c deleted file mode 100644 index b8d67f6a39..0000000000 --- a/ext/wasm/c-pp-lite.c +++ /dev/null @@ -1,2804 +0,0 @@ -/* -** 2022-11-12: -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -** -************************************************************************ -** -** The C-minus Preprocessor: a truly minimal C-like preprocessor. -** Why? Because C preprocessors _can_ process non-C code but generally make -** quite a mess of it. The purpose of this application is an extremely -** minimal preprocessor with only the most basic functionality of a C -** preprocessor, namely. -** -** The supported preprocessor directives are documented in the -** README.md hosted with this file. -** -** Any mention of "#" in the docs, e.g. "#if", is symbolic. The -** directive delimiter is configurable and defaults to "##". Define -** CMPP_DEFAULT_DELIM to a string when compiling to define the default -** at build-time. -** -** This preprocessor has only minimal support for replacement of tokens -** which live in the "content" blocks of inputs (that is, the pieces -** which are not prepocessor lines). -** -** See this file's README.md for details. -** -** Design note: this code makes use of sqlite3. Though not _strictly_ -** needed in order to implement it, this tool was specifically created -** for use with the sqlite3 project's own JavaScript code, so there's -** no reason not to make use of it to do some of the heavy lifting. It -** does not require any cutting-edge sqlite3 features and should be -** usable with any version which supports `WITHOUT ROWID`. -** -** Author(s): -** -** - Stephan Beal -** -** Canonical homes: -** -** - https://fossil.wanderinghorse.net/r/c-pp -** - https://sqlite.org/src/file/ext/wasm/c-pp.c -** -** With the former hosting this app's SCM and the latter being the -** single known deployment of c-pp.c, where much of its development -** happens. -*/ - -#include -#include -#include -#include -#include -#include -#include - -#include "sqlite3.h" - -#if defined(_WIN32) || defined(WIN32) -# include -# include -# ifndef access -# define access(f,m) _access((f),(m)) -# endif -#else -# include -#endif - -#ifndef CMPP_DEFAULT_DELIM -#define CMPP_DEFAULT_DELIM "##" -#endif - -#ifndef CMPP_ATSIGN -#define CMPP_ATSIGN (unsigned char)'@' -#endif - -#if 1 -# define CMPP_NORETURN __attribute__((noreturn)) -#else -# define CMPP_NORETURN -#endif - -/* Fatally exits the app with the given printf-style message. */ -static CMPP_NORETURN void fatalv__base(char const *zFile, int line, - char const *zFmt, va_list); -static CMPP_NORETURN void fatal__base(char const *zFile, int line, - char const *zFmt, ...); -#define fatalv(...) fatalv__base(__FILE__,__LINE__,__VA_ARGS__) -#define fatal(...) fatal__base(__FILE__,__LINE__,__VA_ARGS__) - -/** Proxy for free(), for symmetry with cmpp_realloc(). */ -static void cmpp_free(void *p); -/** A realloc() proxy which dies fatally on allocation error. */ -static void * cmpp_realloc(void * p, unsigned n); -#if 0 -/** A malloc() proxy which dies fatally on allocation error. */ -static void * cmpp_malloc(unsigned n); -#endif - -static void check__oom2(void const *p, char const *zFile, int line){ - if(!p) fatal("Alloc failed at %s:%d", zFile, line); -} -#define check__oom(P) check__oom2((P), __FILE__, __LINE__) - -/* -** If p is stdin or stderr then this is a no-op, else it is a -** proxy for fclose(). This is a no-op if p is NULL. -*/ -static void FILE_close(FILE *p); -/* -** Works like fopen() but accepts the special name "-" to mean either -** stdin (if zMode indicates a real-only mode) or stdout. Fails -** fatally on error. -*/ -static FILE * FILE_open(char const *zName, const char * zMode); -/* -** Reads the entire contents of the given file, allocating it in a -** buffer which gets assigned to `*pOut`. `*nOut` gets assigned the -** length of the output buffer. Fails fatally on error. -*/ -static void FILE_slurp(FILE *pFile, unsigned char **pOut, - unsigned * nOut); - -/* -** Intended to be passed an sqlite3 result code. If it's a non-0 value -** other than SQLITE_ROW or SQLITE_DONE then it emits a fatal error -** message which contains both the given string and the -** sqlite3_errmsg() from the application's database instance. -*/ -static void db_affirm_rc(int rc, const char * zMsg); - -/* -** Proxy for sqlite3_str_finish() which fails fatally if that -** routine returns NULL. -*/ -static char * db_str_finish(sqlite3_str *s, int * n); -/* -** Proxy for sqlite3_str_new() which fails fatally if that -** routine returns NULL. -*/ -static sqlite3_str * db_str_new(void); - -/* -** Proxy for sqlite3_step() which fails fatally if the result -** is anything other than SQLITE_ROW or SQLITE_DONE. -*/ -static int db_step(sqlite3_stmt *pStmt); -/* -** Proxy for sqlite3_bind_int() which fails fatally on error. -*/ -static void db_bind_int(sqlite3_stmt *pStmt, int col, int val); -/* -** Proxy for sqlite3_bind_null() which fails fatally on error. -*/ -static void db_bind_null(sqlite3_stmt *pStmt, int col); -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. -*/ -static void db_bind_text(sqlite3_stmt *pStmt, int col, const char * zStr); -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. -*/ -static void db_bind_textn(sqlite3_stmt *pStmt, int col, const char * zStr, int len); -#if 0 -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. It uses -** sqlite3_str_vappendf() so supports all of its formatting options. -*/ -static void db_bind_textv(sqlite3_stmt *pStmt, int col, const char * zFmt, ...); -#endif -/* -** Proxy for sqlite3_free(), to be passed any memory which is allocated -** by sqlite3_malloc(). -*/ -static void db_free(void *m); - -/* -** Returns true if the first nKey bytes of zKey are a legal string. If -** it returns false and zErrPos is not null, *zErrPos is set to the -** position of the illegal character. If nKey is negative, strlen() is -** used to calculate it. -*/ -static int cmpp_is_legal_key(char const *zKey, int nKey, char const **zErrPos); - -/* -** Fails fatally if !cmpp_is_legal_key(zKey). -*/ -static void cmpp_affirm_legal_key(char const *zKey, int nKey); - -/* -** Adds the given `#define` macro name to the list of macros, ignoring -** any duplicates. Fails fatally on error. -** -** If zVal is NULL then zKey may contain an '=', from which the value -** will be extracted. If zVal is not NULL then zKey may _not_ contain -** an '='. -*/ -static void db_define_add(const char * zKey, char const *zVal); - -/* -** Returns true if the given key is already in the `#define` list, -** else false. Fails fatally on db error. -** -** nName is the length of the key part of zName (which might have -** a following =y part. If it's negative, strlen() is used to -** calculate it. -*/ -static int db_define_has(const char * zName, int nName); - -/* -** Returns true if the given key is already in the `#define` list, and -** it has a truthy value (is not empty and not equal to '0'), else -** false. Fails fatally on db error. -** -** nName is the length of zName, or <0 to use strlen() to figure -** it out. -*/ -static int db_define_get_bool(const char * zName, int nName); - -/* -** Searches for a define where (k GLOB zName). If one is found, a copy -** of it is assigned to *zVal (the caller must eventually db_free() -** it)), *nVal (if nVal is not NULL) is assigned its strlen, and -** returns non-0. If no match is found, 0 is returned and neither -** *zVal nor *nVal are modified. If more than one result matches, a -** fatal error is triggered. -** -** It is legal for *zVal to be NULL (and *nVal to be 0) if it returns -** non-0. That just means that the key was defined with no value part. -*/ -static int db_define_get(const char * zName, int nName, char **zVal, unsigned int *nVal); - -/* -** Removes the given `#define` macro name from the list of -** macros. Fails fatally on error. -*/ -static void db_define_rm(const char * zKey); -/* -** Adds the given filename to the list of being-`#include`d files, -** using the given source file name and line number of error reporting -** purposes. If recursion is later detected. -*/ -static void db_including_add(const char * zKey, const char * zSrc, int srcLine); -/* -** Adds the given dir to the list of includes. They are checked in the -** order they are added. -*/ -static void db_include_dir_add(const char * zKey); -/* -** Returns a resolved path of PREFIX+'/'+zKey, where PREFIX is one of -** the `#include` dirs (db_include_dir_add()). If no file match is -** found, NULL is returned. Memory must eventually be passed to -** db_free() to free it. -*/ -static char * db_include_search(const char * zKey); -/* -** Removes the given key from the `#include` list. -*/ -static void db_include_rm(const char * zKey); -/* -** A proxy for sqlite3_prepare() which fails fatally on error. -*/ -static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); - -/* -** Opens the given file and processes its contents as c-pp, sending -** all output to the global c-pp output channel. Fails fatally on -** error. If bRaw is true then the file's contents are passed through -** verbatim, rather than being preprocessed. -*/ -static void cmpp_process_file(const char * zName, int bRaw); - -/* -** Operator policy for cmpp_kvp_parse(). -*/ -enum cmpp_key_op_e { - /* Fail if the key contains an operator. */ - cmpp_key_op_none, - /* Accept only '='. */ - cmpp_key_op_eq1 -}; -typedef enum cmpp_key_op_e cmpp_key_op_e; - -/* -** Operators and operator policies for use with X=Y-format keys. -*/ -#define cmpp_kvp_op_map(E) \ - E(none,"") \ - E(eq1,"=") \ - E(eq2,"==") \ - E(lt,"<") \ - E(le,"<=") \ - E(gt,">") \ - E(ge,">=") - -enum cmpp_kvp_op_e { -#define E(N,S) cmpp_kvp_op_ ## N, - cmpp_kvp_op_map(E) -#undef E -}; -typedef enum cmpp_kvp_op_e cmpp_kvp_op_e; - -/* -** A snippet from a string. -*/ -struct cmpp_snippet { - char const *z; - unsigned int n; -}; -typedef struct cmpp_snippet cmpp_snippet; -#define cmpp_snippet_empty_m {0,0} - -/* -** Result type for cmpp_kvp_parse(). -*/ -struct cmpp_kvp { - cmpp_snippet k; - cmpp_snippet v; - cmpp_kvp_op_e op; -}; - -typedef struct cmpp_kvp cmpp_kvp; -#define cmpp_kvp_empty_m \ - {cmpp_snippet_empty_m,cmpp_snippet_empty_m,cmpp_kvp_op_none} -static const cmpp_kvp cmpp_kvp_empty = cmpp_kvp_empty_m; - -/* -** Parses X or X=Y into p. Fails fatally on error. -** -** If nKey is negative then strlen() is used to calculate it. -** -** The third argument specifies whether/how to permit/treat the '=' -** part of X=Y. -*/ -static void cmpp_kvp_parse(cmpp_kvp * p, - char const *zKey, int nKey, - cmpp_kvp_op_e opPolicy); - -/* -** Wrapper around a FILE handle. -*/ -typedef struct FileWrapper FileWrapper; -struct FileWrapper { - /* File's name. */ - char const *zName; - /* FILE handle. */ - FILE * pFile; - /* Where FileWrapper_slurp() stores the file's contents. */ - unsigned char * zContent; - /* Size of this->zContent, as set by FileWrapper_slurp(). */ - unsigned nContent; - /* See Global::pFiles. */ - FileWrapper * pTail; -}; -#define FileWrapper_empty_m {0,0,0,0,0} -static const FileWrapper FileWrapper_empty = FileWrapper_empty_m; - -/* -** Proxy for FILE_close() and frees all memory owned by p. A no-op if -** p is already closed. -*/ -static void FileWrapper_close(FileWrapper * p); -/* Proxy for FILE_open(). Closes p first if it's currently opened. */ -static void FileWrapper_open(FileWrapper * p, const char * zName, const char *zMode); -/* Proxy for FILE_slurp(). */ -static void FileWrapper_slurp(FileWrapper * p); -/* -** If p->zContent ends in \n or \r\n, that part is replaced with 0 and -** p->nContent is adjusted. Returns true if it chomps, else false. -*/ -int FileWrapper_chomp(FileWrapper * p); - -/* -** Outputs a printf()-formatted message to stderr. -*/ -static void g_stderr(char const *zFmt, ...); -/* -** Outputs a printf()-formatted message to stderr. -*/ -static void g_stderrv(char const *zFmt, va_list); -#define g_debug(lvl,pfexpr) \ - if(lvl<=g.flags.doDebug) g_stderr("%s @ %s():%d: ",g.zArgv0,__func__,__LINE__); \ - if(lvl<=g.flags.doDebug) g_stderr pfexpr - -#define g_warn(zFmt,...) g_stderr("%s:%d %s() " zFmt "\n", __FILE__, __LINE__, __func__, __VA_ARGS__) -#define g_warn0(zMsg) g_stderr("%s:%d %s() %s\n", __FILE__, __LINE__, __func__, zMsg) - -void cmpp_free(void *p){ - sqlite3_free(p); -} - -void * cmpp_realloc(void * p, unsigned n){ - void * const rc = sqlite3_realloc(p, n); - if(!rc) fatal("realloc(P,%u) failed", n); - return rc; -} - -#if 0 -void * cmpp_malloc(unsigned n){ - void * const rc = sqlite3_alloc(n); - if(!rc) fatal("malloc(%u) failed", n); - return rc; -} -#endif - -FILE * FILE_open(char const *zName, const char * zMode){ - FILE * p; - if('-'==zName[0] && 0==zName[1]){ - p = strstr(zMode,"w") ? stdout : stdin; - }else{ - p = fopen(zName, zMode); - if(!p) fatal("Cannot open file [%s] with mode [%s]", zName, zMode); - } - return p; -} - -void FILE_close(FILE *p){ - if(p && p!=stdout && p!=stderr){ - fclose(p); - } -} - -void FILE_slurp(FILE *pFile, unsigned char **pOut, - unsigned * nOut){ - unsigned char zBuf[1024 * 8]; - unsigned char * pDest = 0; - unsigned nAlloc = 0; - unsigned nOff = 0; - /* Note that this needs to be able to work on non-seekable streams, - ** thus we read in chunks instead of doing a single alloc and - ** filling it in one go. */ - while( !feof(pFile) ){ - size_t const n = fread(zBuf, 1, sizeof(zBuf), pFile); - if(n>0){ - if(nAlloc < nOff + n + 1){ - nAlloc = nOff + n + 1; - pDest = cmpp_realloc(pDest, nAlloc); - } - memcpy(pDest + nOff, zBuf, n); - nOff += n; - } - } - if(pDest) pDest[nOff] = 0; - *pOut = pDest; - *nOut = nOff; -} - -void FileWrapper_close(FileWrapper * p){ - if(p->pFile) FILE_close(p->pFile); - if(p->zContent) cmpp_free(p->zContent); - *p = FileWrapper_empty; -} - -void FileWrapper_open(FileWrapper * p, const char * zName, - const char * zMode){ - FileWrapper_close(p); - p->pFile = FILE_open(zName, zMode); - p->zName = zName; -} - -void FileWrapper_slurp(FileWrapper * p){ - assert(!p->zContent); - assert(p->pFile); - FILE_slurp(p->pFile, &p->zContent, &p->nContent); -} - -int FileWrapper_chomp(FileWrapper * p){ - if( p->nContent && '\n'==p->zContent[p->nContent-1] ){ - p->zContent[--p->nContent] = 0; - if( p->nContent && '\r'==p->zContent[p->nContent-1] ){ - p->zContent[--p->nContent] = 0; - } - return 1; - } - return 0; -} - -enum CmppParseState { -TS_Start = 1, -TS_If, -TS_IfPassed, -TS_Else, -TS_Error -}; -typedef enum CmppParseState CmppParseState; -enum CmppTokenType { - -#define CmppToken_map(E) \ - E(Invalid,0) \ - E(Assert,"assert") \ - E(AtPolicy,"@policy") \ - E(Comment,"//") \ - E(Define,"define") \ - E(Elif,"elif") \ - E(Else,"else") \ - E(Endif,"endif") \ - E(Error,"error") \ - E(If,"if") \ - E(Include,"include") \ - E(Line,0) \ - E(Opaque,0) \ - E(Pragma,"pragma") \ - E(Savepoint,"savepoint") \ - E(Stderr,"stderr") \ - E(Undef,"undef") - -#define E(N,TOK) TT_ ## N, - CmppToken_map(E) -#undef E -}; -typedef enum CmppTokenType CmppTokenType; - -/* -** Map of directive (formerly keyword) names and their token types. -*/ -static const struct { -#define E(N,TOK) struct cmpp_snippet N; - CmppToken_map(E) -#undef E -} DStrings = { -#define E(N,TOK) .N = {TOK,sizeof(TOK)-1}, - CmppToken_map(E) -#undef E -}; - -//static -char const * TT_cstr(int tt){ - switch(tt){ -#define E(N,TOK) case TT_ ## N: return DStrings.N.z; - CmppToken_map(E) -#undef E - } - return NULL; -} - -struct CmppToken { - CmppTokenType ttype; - /* Line number of this token in the source file. */ - unsigned lineNo; - /* Start of the token. */ - unsigned char const * zBegin; - /* One-past-the-end byte of the token. */ - unsigned char const * zEnd; -}; -typedef struct CmppToken CmppToken; -#define CmppToken_empty_m {TT_Invalid,0,0,0} -static const CmppToken CmppToken_empty = CmppToken_empty_m; - -/* -** CmppLevel represents one "level" of tokenization, starting at the -** top of the main input, incrementing once for each level of `#if`, -** and decrementing for each `#endif`. -** pushes a level. -*/ -typedef struct CmppLevel CmppLevel; -struct CmppLevel { - unsigned short flags; - /* - ** Used for controlling which parts of an if/elif/...endif chain - ** should get output. - */ - unsigned short skipLevel; - /* The token which started this level (an 'if' or 'include'). */ - CmppToken token; - CmppParseState pstate; -}; -#define CmppLevel_empty_m {0U,0U,CmppToken_empty_m,TS_Start} -static const CmppLevel CmppLevel_empty = CmppLevel_empty_m; -enum CmppLevel_Flags { -/* Max depth of nested `#if` constructs in a single tokenizer. */ -CmppLevel_Max = 10, -/* Max number of keyword arguments. */ -CmppArgs_Max = 15, -/* Directive line buffer size */ -CmppArgs_BufSize = 1024, -/* Flag indicating that output for a CmpLevel should be elided. */ -CmppLevel_F_ELIDE = 0x01, -/* -** Mask of CmppLevel::flags which are inherited when CmppLevel_push() -** is used. -*/ -CmppLevel_F_INHERIT_MASK = CmppLevel_F_ELIDE -}; - -typedef struct CmppTokenizer CmppTokenizer; -typedef struct CmppKeyword CmppKeyword; -typedef void (*cmpp_keyword_f)(CmppKeyword const * pKw, CmppTokenizer * t); -struct CmppKeyword { - const char *zName; - unsigned nName; - int bTokenize; - CmppTokenType ttype; - cmpp_keyword_f xCall; -}; - -static CmppKeyword const * CmppKeyword_search(const char *zName); -static void cmpp_process_keyword(CmppTokenizer * const t); - -/* -** Tokenizer for c-pp input files. -*/ -struct CmppTokenizer { - const char * zName; /* Input (file) name for error reporting */ - unsigned const char * zBegin; /* start of input */ - unsigned const char * zEnd; /* one-after-the-end of input */ - unsigned const char * zPos; /* current position */ - unsigned int lineNo; /* line # of current pos */ - unsigned nSavepoint; - CmppParseState pstate; - CmppToken token; /* current token result */ - struct { - unsigned ndx; - CmppLevel stack[CmppLevel_Max]; - } level; - /* Args for use in cmpp_keyword_f() impls. */ - struct { - CmppKeyword const * pKw; - int argc; - const unsigned char * argv[CmppArgs_Max]; - unsigned char lineBuf[CmppArgs_BufSize]; - } args; -}; -#define CT_level(t) (t)->level.stack[(t)->level.ndx] -#define CT_pstate(t) CT_level(t).pstate -#define CT_skipLevel(t) CT_level(t).skipLevel -#define CLvl_skip(lvl) ((lvl)->skipLevel || ((lvl)->flags & CmppLevel_F_ELIDE)) -#define CT_skip(t) CLvl_skip(&CT_level(t)) -#define CmppTokenizer_empty_m { \ - .zName=0, .zBegin=0, .zEnd=0, \ - .zPos=0, \ - .lineNo=1U, \ - .pstate = TS_Start, \ - .token = CmppToken_empty_m, \ - .level = {0U,{CmppLevel_empty_m}}, \ - .args = {0,0,{0},{0}} \ - } -static const CmppTokenizer CmppTokenizer_empty = CmppTokenizer_empty_m; - -static void CmppTokenizer_cleanup(CmppTokenizer * const t); - -static void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n); -/*static void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...);*/ - -/* -** Pushes a new level into the given tokenizer. Fails fatally if -** it's too deep. -*/ -static void CmppLevel_push(CmppTokenizer * const t); -/* -** Pops a level from the tokenizer. Fails fatally if the top -** level is popped. -*/ -static void CmppLevel_pop(CmppTokenizer * const t); -/* -** Returns the current level object. -*/ -static CmppLevel * CmppLevel_get(CmppTokenizer * const t); - -/* -** Policies for how to handle undefined @tokens@ when performing -** content filtering. -*/ -enum AtPolicy { - AT_invalid = -1, - /** Turn off @foo@ parsing. */ - AT_OFF = 0, - /** Retain undefined @foo@ - emit it as-is. */ - AT_RETAIN, - /** Elide undefined @foo@. */ - AT_ELIDE, - /** Error for undefined @foo@. */ - AT_ERROR, - AT_DEFAULT = AT_ERROR -}; -typedef enum AtPolicy AtPolicy; - -static AtPolicy AtPolicy_fromStr(char const *z, int bEnforce){ - if( 0==strcmp(z, "retain") ) return AT_RETAIN; - if( 0==strcmp(z, "elide") ) return AT_ELIDE; - if( 0==strcmp(z, "error") ) return AT_ERROR; - if( 0==strcmp(z, "off") ) return AT_OFF; - if( bEnforce ){ - fatal("Invalid @ policy value: %s. " - "Try one of retain|elide|error|off.", z); - } - return AT_invalid; -} - -/* -** Global app state singleton. -*/ -static struct Global { - /* main()'s argv[0]. */ - const char * zArgv0; - /* App's db instance. */ - sqlite3 * db; - /* Current tokenizer (for error reporting purposes). */ - CmppTokenizer const * tok; - /* - ** We use a linked-list of these to keep track of our opened - ** files so that we can clean then up via atexit() in the case of - ** fatal error (to please valgrind). - */ - FileWrapper * pFiles; - /* Output channel. */ - FileWrapper out; - struct { - /* - ** Bytes of the keyword delimiter/prefix. Owned - ** elsewhere. - */ - const char * z; - /* Byte length of this->zDelim. */ - unsigned short n; - /* - ** The @token@ delimiter. - ** - ** Potential TODO is replace this with a pair of opener/closer - ** strings, e.g. "{{" and "}}". - */ - const unsigned char chAt; - } delim; - struct { -#define CMPP_SAVEPOINT_NAME "_cmpp_" -#define GStmt_map(E) \ - E(defIns,"INSERT OR REPLACE INTO def(k,v) VALUES(?,?)") \ - E(defDel,"DELETE FROM def WHERE k GLOB ?") \ - E(defHas,"SELECT 1 FROM def WHERE k GLOB ?") \ - E(defGet,"SELECT k,v FROM def WHERE k GLOB ?") \ - E(defGetBool, \ - "SELECT 1 FROM def WHERE k = ?1" \ - " AND v IS NOT NULL" \ - " AND '0'!=v AND ''!=v") \ - E(defSelAll,"SELECT k,v FROM def ORDER BY k") \ - E(inclIns,"INSERT OR FAIL INTO incl(file,srcFile," \ - "srcLine) VALUES(?,?,?)") \ - E(inclDel,"DELETE FROM incl WHERE file=?") \ - E(inclHas,"SELECT 1 FROM incl WHERE file=?") \ - E(inclPathAdd,"INSERT OR FAIL INTO " \ - "inclpath(seq,dir) VALUES(?,?)") \ - E(inclSearch, \ - "SELECT ?1 fn WHERE fileExists(fn) " \ - "UNION ALL SELECT * FROM (" \ - "SELECT replace(dir||'/'||?1, '//','/') AS fn " \ - "FROM inclpath WHERE fileExists(fn) ORDER BY seq"\ - ")") \ - E(spBegin,"SAVEPOINT " CMPP_SAVEPOINT_NAME) \ - E(spRollback,"ROLLBACK TO SAVEPOINT " \ - CMPP_SAVEPOINT_NAME) \ - E(spRelease,"RELEASE SAVEPOINT " CMPP_SAVEPOINT_NAME) - -#define E(N,S) sqlite3_stmt * N; - GStmt_map(E) -#undef E - } stmt; - struct { - FILE * pFile; - int expandSql; - } sqlTrace; - struct { - AtPolicy atPolicy; - /* If true, enables certain debugging output. */ - char doDebug; - /* If true, chomp() files read via -Fx=file. */ - char chompF; - } flags; -} g = { - .zArgv0 = "?", - .db = 0, - .tok = 0, - .pFiles = 0, - .out = FileWrapper_empty_m, - .delim = { - .z = CMPP_DEFAULT_DELIM, - .n = (unsigned short) sizeof(CMPP_DEFAULT_DELIM)-1, - .chAt = '@' - }, - .stmt = { - .defIns = 0, - .defDel = 0, - .defHas = 0, - .defGet = 0, - .defGetBool = 0, - .inclIns = 0, - .inclDel = 0, - .inclHas = 0, - .inclPathAdd = 0, - .inclSearch = 0 - }, - .sqlTrace = { - .pFile = 0, - .expandSql = 0 - }, - .flags = { - .atPolicy = AT_OFF, - .doDebug = 0, - .chompF = 0 - } -}; - -/** Distinct IDs for each g.stmt member. */ -enum GStmt_e { - GStmt_none = 0, -#define E(N,S) GStmt_ ## N, - GStmt_map(E) -#undef E -}; - -/* -** Returns the g.stmt.X corresponding to `which`, initializing it if -** needed. It does not return NULL - it fails fatally on error. -*/ -static sqlite3_stmt * g_stmt(enum GStmt_e which){ - sqlite3_stmt ** q = 0; - char const * zSql = 0; - switch(which){ - case GStmt_none: - fatal("GStmt_none is not a valid statement handle"); - return NULL; -#define E(N,S) case GStmt_ ## N: zSql = S; q = &g.stmt.N; break; - GStmt_map(E) -#undef E - } - assert( q ); - assert( zSql && *zSql ); - if( !*q ){ - db_prepare(q, "%s", zSql); - assert( *q ); - } - return *q; -} -static void g_stmt_reset(sqlite3_stmt * const q){ - sqlite3_clear_bindings(q); - sqlite3_reset(q); -} - -#if 0 -/* -** Outputs a printf()-formatted message to c-pp's global output -** channel. -*/ -static void g_outf(char const *zFmt, ...); -void g_outf(char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - vfprintf(g.out.pFile, zFmt, va); - va_end(va); -} -#endif - -/* Outputs n bytes from z to c-pp's global output channel. */ -static void g_out(void const *z, unsigned int n); -void g_out(void const *z, unsigned int n){ - if(g.out.pFile && 1!=fwrite(z, n, 1, g.out.pFile)){ - int const err = errno; - fatal("fwrite() output failed with errno #%d", err); - } -} - -void g_stderrv(char const *zFmt, va_list va){ - if( g.out.pFile==stdout ){ - fflush(g.out.pFile); - } - vfprintf(stderr, zFmt, va); -} - -void g_stderr(char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - g_stderrv(zFmt, va); - va_end(va); -} - -/* -** Emits n bytes of z if CT_skip(t) is false. -*/ -void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n){ - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); - if(!CT_skip(t)) g_out(z, n); -} - -void CmppLevel_push(CmppTokenizer * const t){ - CmppLevel * pPrev; - CmppLevel * p; - if(t->level.ndx+1 == (unsigned)CmppLevel_Max){ - fatal("%sif nesting level is too deep. Max=%d\n", - g.delim.z, CmppLevel_Max); - } - pPrev = &CT_level(t); - g_debug(3,("push from tokenizer level=%u flags=%04x\n", - t->level.ndx, pPrev->flags)); - p = &t->level.stack[++t->level.ndx]; - *p = CmppLevel_empty; - p->token = t->token; - p->flags = (CmppLevel_F_INHERIT_MASK & pPrev->flags); - if(CLvl_skip(pPrev)) p->flags |= CmppLevel_F_ELIDE; - g_debug(3,("push to tokenizer level=%u flags=%04x\n", - t->level.ndx, p->flags)); -} - -void CmppLevel_pop(CmppTokenizer * const t){ - if(!t->level.ndx){ - fatal("Internal error: CmppLevel_pop() at the top of the stack"); - } - g_debug(3,("pop from tokenizer level=%u, flags=%04x skipLevel?=%d\n", - t->level.ndx, - t->level.stack[t->level.ndx].flags, CT_skipLevel(t))); - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); - t->level.stack[t->level.ndx--] = CmppLevel_empty; - g_debug(3,("pop to tokenizer level=%u, flags=%04x\n", t->level.ndx, - t->level.stack[t->level.ndx].flags)); - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); -} - -CmppLevel * CmppLevel_get(CmppTokenizer * const t){ - return &t->level.stack[t->level.ndx]; -} - - -void db_affirm_rc(int rc, const char * zMsg){ - switch(rc){ - case 0: - case SQLITE_DONE: - case SQLITE_ROW: - break; - default: - assert( g.db ); - fatal("Db error #%d %s: %s", rc, zMsg, - sqlite3_errmsg(g.db)); - } -} - -int db_step(sqlite3_stmt *pStmt){ - int const rc = sqlite3_step(pStmt); - switch( rc ){ - case SQLITE_ROW: - case SQLITE_DONE: - break; - default: - db_affirm_rc(rc, "from db_step()"); - } - return rc; -} - -static sqlite3_str * db_str_new(void){ - sqlite3_str * rc = sqlite3_str_new(g.db); - if(!rc) fatal("Alloc failed for sqlite3_str_new()"); - return rc; -} - -static char * db_str_finish(sqlite3_str *s, int * n){ - int const rc = sqlite3_str_errcode(s); - if(rc) fatal("Error #%d from sqlite3_str_errcode()", rc); - if(n) *n = sqlite3_str_length(s); - char * z = sqlite3_str_finish(s); - if(!z) fatal("Alloc failed for sqlite3_str_new()"); - return z; -} - -void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ - int rc; - sqlite3_str * str = db_str_new(); - char * z = 0; - int n = 0; - va_list va; - va_start(va, zSql); - sqlite3_str_vappendf(str, zSql, va); - va_end(va); - rc = sqlite3_str_errcode(str); - if(rc) fatal("sqlite3_str_errcode() = %d", rc); - z = db_str_finish(str, &n); - rc = sqlite3_prepare_v2(g.db, z, n, pStmt, 0); - if(rc) fatal("Error #%d (%s) preparing: %s", - rc, sqlite3_errmsg(g.db), z); - sqlite3_free(z); -} - -void db_bind_int(sqlite3_stmt *pStmt, int col, int val){ - db_affirm_rc(sqlite3_bind_int(pStmt, col, val), - "from db_bind_int()"); -} - -void db_bind_null(sqlite3_stmt *pStmt, int col){ - db_affirm_rc(sqlite3_bind_null(pStmt, col), - "from db_bind_null()"); -} - -void db_bind_textn(sqlite3_stmt *pStmt, int col, - const char * zStr, int n){ - db_affirm_rc( - (zStr && n) - ? sqlite3_bind_text(pStmt, col, zStr, n, SQLITE_TRANSIENT) - : sqlite3_bind_null(pStmt, col), - "from db_bind_textn()" - ); -} - -void db_bind_text(sqlite3_stmt *pStmt, int col, - const char * zStr){ - db_bind_textn(pStmt, col, zStr, -1); -} - -#if 0 -void db_bind_textv(sqlite3_stmt *pStmt, int col, - const char * zFmt, ...){ - int rc; - sqlite3_str * str = db_str_new(); - int n = 0; - char * z; - va_list va; - va_start(va,zFmt); - sqlite3_str_vappendf(str, zFmt, va); - va_end(va); - z = db_str_finish(str, &n); - rc = sqlite3_bind_text(pStmt, col, z, n, sqlite3_free); - db_affirm_rc(rc,"from db_bind_textv()"); -} -#endif - -void db_free(void *m){ - sqlite3_free(m); -} - -void db_define_add(const char * zKey, char const *zVal){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, - zVal - ? cmpp_key_op_none - : cmpp_key_op_eq1 - ); - if( kvp.v.z ){ - if( zVal ){ - assert(!"cannot happen - cmpp_key_op_none will prevent it"); - fatal("Cannot assign two values to [%.*s] [%.*s] [%s]", - kvp.k.n, kvp.k.z, kvp.v.n, kvp.v.z, zVal); - } - }else{ - kvp.v.z = zVal; - kvp.v.n = zVal ? (int)strlen(zVal) : 0; - } - sqlite3_stmt * const q = g_stmt(GStmt_defIns); - //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); - db_bind_textn(q, 1, kvp.k.z, kvp.k.n); - if( kvp.v.z ){ - if( kvp.v.n ){ - db_bind_textn(q, 2, kvp.v.z, (int)kvp.v.n); - }else{ - db_bind_null(q, 2); - } - }else{ - db_bind_int(q, 2, 1); - } - db_step(q); - g_debug(2,("define: %s%s%s\n", - zKey, - zVal ? " with value " : "", - zVal ? zVal : "")); - sqlite3_reset(q); -} - -static void db_define_add_file(const char * zKey){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, cmpp_kvp_op_eq1); - if( !kvp.v.z || !kvp.v.n ){ - fatal("Invalid filename: %s", zKey); - } - sqlite3_stmt * q = 0; - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, kvp.v.z, "r"); - FileWrapper_slurp(&fw); - q = g_stmt(GStmt_defIns); - //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); - db_bind_textn(q, 1, kvp.k.z, (int)kvp.k.n); - if( g.flags.chompF ){ - FileWrapper_chomp(&fw); - } - if( fw.nContent ){ - db_affirm_rc( - sqlite3_bind_text(q, 2, - (char const *)fw.zContent, - (int)fw.nContent, sqlite3_free), - "binding file content"); - fw.zContent = 0 /* transfered ownership */; - fw.nContent = 0; - }else{ - db_affirm_rc( sqlite3_bind_null(q, 2), - "binding empty file content"); - } - FileWrapper_close(&fw); - db_step(q); - g_stmt_reset(q); - g_debug(2,("define: %s%s%s\n", - kvp.k.z, - kvp.v.z ? " with value " : "", - kvp.v.z ? kvp.v.z : "")); -} - -#define ustr_c(X) ((unsigned char const *)X) - -static inline unsigned int cmpp_strlen(char const *z, int n){ - return n<0 ? (int)strlen(z) : (unsigned)n; -} - - -int db_define_has(const char * zName, int nName){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_defHas); - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(1,("defined [%s] ?= %d\n",zName, rc)); - g_stmt_reset(q); - return rc; -} - -int db_define_get_bool(const char * zName, int nName){ - sqlite3_stmt * const q = g_stmt(GStmt_defGetBool); - int rc = 0; - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - if( SQLITE_ROW==sqlite3_step(q) ){ - fatal("Key is ambiguous: %s", zName); - } - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_stmt_reset(q); - return rc; -} - -int db_define_get(const char * zName, int nName, - char **zVal, unsigned int *nVal){ - sqlite3_stmt * q = g_stmt(GStmt_defGet); - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - int n = 0; - int rc = db_step(q); - if(SQLITE_ROW == rc){ - const unsigned char * z = sqlite3_column_text(q, 1); - n = sqlite3_column_bytes(q,1); - if( nVal ) *nVal = (unsigned)n; - *zVal = sqlite3_mprintf("%.*s", n, z); - if( n && z ) check__oom(*zVal); - if( SQLITE_ROW==sqlite3_step(q) ){ - db_free(*zVal); - *zVal = 0; - fatal("Key is ambiguous: %.*s\n", - nName, zName); - } - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(1,("define [%.*s] ?= %d %.*s\n", - nName, zName, rc, - *zVal ? n : 0, - *zVal ? *zVal : "")); - g_stmt_reset(q); - return rc; -} - -void db_define_rm(const char * zKey){ - int rc; - int n = 0; - sqlite3_stmt * const q = g_stmt(GStmt_defDel); - db_bind_text(q, 1, zKey); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping DELETE on def"); - } - g_debug(2,("undefine: %.*s\n",n, zKey)); - g_stmt_reset(q); -} - -void db_including_add(const char * zKey, const char * zSrc, int srcLine){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclIns); - db_bind_text(q, 1, zKey); - db_bind_text(q, 2, zSrc); - db_bind_int(q, 3, srcLine); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping INSERT on incl"); - } - g_debug(2,("is-including-file add [%s] from [%s]:%d\n", zKey, zSrc, srcLine)); - g_stmt_reset(q); -} - -void db_include_rm(const char * zKey){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclDel); - db_bind_text(q, 1, zKey); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping DELETE on incl"); - } - g_debug(2,("inclpath rm [%s]\n", zKey)); - g_stmt_reset(q); -} - -char * db_include_search(const char * zKey){ - char * zName = 0; - sqlite3_stmt * const q = g_stmt(GStmt_inclSearch); - db_bind_text(q, 1, zKey); - if(SQLITE_ROW==db_step(q)){ - const unsigned char * z = sqlite3_column_text(q, 0); - zName = z ? sqlite3_mprintf("%s", z) : 0; - if(!zName) fatal("Alloc failed"); - } - g_stmt_reset(q); - return zName; -} - -static int db_including_has(const char * zName){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclHas); - db_bind_text(q, 1, zName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(2,("inclpath has [%s] = %d\n",zName, rc)); - g_stmt_reset(q); - return rc; -} - -#if 0 -/* -** Fails fatally if the `#include` list contains the given key. -*/ -static void db_including_check(const char * zKey); -void db_including_check(const char * zName){ - if(db_including_has(zName)){ - fatal("Recursive include detected: %s\n", zName); - } -} -#endif - -void db_include_dir_add(const char * zDir){ - static int seq = 0; - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclPathAdd); - db_bind_int(q, 1, ++seq); - db_bind_text(q, 2, zDir); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping INSERT on inclpath"); - } - g_debug(2,("inclpath add #%d: %s\n",seq, zDir)); - g_stmt_reset(q); -} - -void g_FileWrapper_link(FileWrapper *fp){ - assert(!fp->pTail); - fp->pTail = g.pFiles; - g.pFiles = fp; -} - -void g_FileWrapper_close(FileWrapper *fp){ - assert(fp); - assert(fp->pTail || g.pFiles==fp); - g.pFiles = fp->pTail; - fp->pTail = 0; - FileWrapper_close(fp); -} - -static void g_cleanup(int bCloseFileChain){ - if( g.db ){ -#define E(N,S) sqlite3_finalize(g.stmt.N); g.stmt.N = 0; - GStmt_map(E) -#undef E - } - if( bCloseFileChain ){ - FileWrapper * fpNext = 0; - for( FileWrapper * fp=g.pFiles; fp; fp=fpNext ){ - fpNext = fp->pTail; - fp->pTail = 0; - FileWrapper_close(fp); - } - } - FileWrapper_close(&g.out); - if(g.db){ - sqlite3_close(g.db); - g.db = 0; - } -} - -static void cmpp_atexit(void){ - g_cleanup(1); -} - -int cmpp_is_legal_key(char const *zKey, int nKey, char const **zAt){ - char const * z = zKey; - nKey = cmpp_strlen(zKey, nKey); - if( !nKey ){ - if( zAt ) *zAt = z; - return 0; - } - char const * const zEnd = z ? z + nKey : NULL; - for( ; z < zEnd; ++z ){ - switch( (0x80 & *z) ? 0 : *z ){ - case 0: - case '_': - continue; - case '-': - case '.': - case '/': - case ':': - case '=': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - if( z==zKey ) break; - continue; - default: - if( isalpha((int)*z) ) continue; - } - if( zAt ) *zAt = z; - return 0; - } - assert( z==zEnd ); - return 1; -} - -void cmpp_affirm_legal_key(char const *zKey, int nKey){ - char const *zAt = 0; - nKey = cmpp_strlen(zKey, nKey); - if( !cmpp_is_legal_key(zKey, nKey, &zAt) ){ - assert( zAt ); - fatal("Illegal character 0x%02x in key [%.*s]\n", - (int)*zAt, nKey, zKey); - } -} - -/* -** sqlite3 UDF which returns true if its argument refers to an -** accessible file, else false. -*/ -static void udf_file_exists( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName; - (void)(argc); /* Unused parameter */ - zName = (const char*)sqlite3_value_text(argv[0]); - if( zName==0 ) return; - sqlite3_result_int(context, 0==access(zName, 0)); -} - -/** - ** This sqlite3_trace_v2() callback outputs tracing info using - ** g.sqlTrace.pFile. -*/ -static int cmpp__db_sq3TraceV2(unsigned t,void*c,void*p,void*x){ - static unsigned int counter = 0; - switch(t){ - case SQLITE_TRACE_STMT:{ - FILE * const fp = g.sqlTrace.pFile; - if( fp ){ - char const * const zSql = (char const *)x; - char * const zExp = g.sqlTrace.expandSql - ? sqlite3_expanded_sql((sqlite3_stmt*)p) - : 0; - fprintf(fp, "SQL TRACE #%u: %s\n", - ++counter, zExp ? zExp : zSql); - sqlite3_free(zExp); - } - break; - } - } - return 0; -} - -/* Initialize g.db, failing fatally on error. */ -static void cmpp_initdb(void){ - int rc; - char * zErr = 0; - const char * zSchema = - "CREATE TABLE def(" - /* ^^^ defines */ - "k TEXT PRIMARY KEY NOT NULL," - "v TEXT DEFAULT NULL" - ") WITHOUT ROWID;" - "CREATE TABLE incl(" - /* ^^^ files currently being included */ - "file TEXT PRIMARY KEY NOT NULL," - "srcFile TEXT DEFAULT NULL," - "srcLine INTEGER DEFAULT 0" - ") WITHOUT ROWID;" - "CREATE TABLE inclpath(" - /* ^^^ include path */ - "seq INTEGER UNIQUE ON CONFLICT IGNORE, " - "dir TEXT PRIMARY KEY NOT NULL ON CONFLICT IGNORE" - ");" - "BEGIN;" - ; - assert(0==g.db); - if(g.db) return; - rc = sqlite3_open_v2(":memory:", &g.db, SQLITE_OPEN_READWRITE, 0); - if(rc) fatal("Error opening :memory: db."); - sqlite3_trace_v2(g.db, SQLITE_TRACE_STMT, cmpp__db_sq3TraceV2, 0); - rc = sqlite3_exec(g.db, zSchema, 0, 0, &zErr); - if(rc) fatal("Error initializing database: %s", zErr); - rc = sqlite3_create_function(g.db, "fileExists", 1, - SQLITE_UTF8|SQLITE_DIRECTONLY, 0, - udf_file_exists, 0, 0); - db_affirm_rc(rc, "UDF registration failed."); -} - -/* -** For position zPos, which must be in the half-open range -** [zBegin,zEnd), returns g.delim.n if it is at the start of a line and -** starts with g.delim.z, else returns 0. -*/ -//static -unsigned short cmpp_is_delim(unsigned char const *zBegin, - unsigned char const *zEnd, - unsigned char const *zPos){ - assert(zEnd>zBegin); - assert(zPos=zBegin); - if(zPos>zBegin && - ('\n'!=*(zPos - 1) - || ((unsigned)(zEnd - zPos) <= g.delim.n))){ - return 0; - }else if(0==memcmp(zPos, g.delim.z, g.delim.n)){ - return g.delim.n; - }else{ - return 0; - } -} - -static void cmpp_t_out_expand(CmppTokenizer * const t, - unsigned char const * zFrom, - unsigned int n); - -static inline int cmpp__isspace(int ch){ - return ' '==ch || '\t'==ch; -} - -static inline unsigned cmpp__strlenu(unsigned char const *z, int n){ - return n<0 ? (unsigned)strlen((char const *)z) : (unsigned)n; -} - -static inline void cmpp__skip_space_c( unsigned char const **p, - unsigned char const *zEnd ){ - unsigned char const * z = *p; - while( zzPos,t->zEnd) for a derective delimiter. Emits any - non-delimiter output found along the way. - - This updtes t->zPos and t->lineNo as it goes. - - If a delimiter is found, it updates t->token and returns 0. - On no match returns 0. -*/ -static -int CmppTokenizer__delim_search(CmppTokenizer * const t){ - if(!t->zPos) t->zPos = t->zBegin; - if( t->zPos>=t->zEnd ){ - return 0; - } - assert( (t->zPos==t->zBegin || t->zPos[-1]=='\n') - && "Else we've mismanaged something."); - char const * const zD = g.delim.z; - unsigned short const nD = g.delim.n; - unsigned char const * const zEnd = t->zEnd; - unsigned char const * zLeft = t->zPos; - unsigned char const * z = zLeft; - - assert( 0==*zEnd && "Else we'll misinteract with strcspn()" ); - if( *zEnd ){ - fatal("Input must be NUL-terminated."); - return 0; - } -#define tflush \ - if(z>zEnd) z=zEnd; \ - if( z>zLeft ) { \ - cmpp_t_out_expand(t, zLeft, (unsigned)(z-zLeft)); \ - } zLeft = z - while(z < zEnd){ - size_t nNlTotal = 0; - unsigned char const * zNl; - size_t nNl2 = strcspn((char const *)z, "\n"); - zNl = (z + nNl2 >= zEnd ? zEnd : z + nNl2); - if( nNl2 >= CmppArgs_BufSize /* too long */ - //|| '\n'!=(char)*zNl /* end of input */ - /* ^^^ we have to accept a missing trailing EOL for the - sake of -e scripts. */ - ){ - /* we'd like to error out here, but only if we know we're - reading reading a directive line. */ - ++t->lineNo; - z = zNl + 1; - tflush; - continue; - } - nNlTotal += nNl2; - assert( '\n'==*zNl || !*zNl ); - assert( '\n'==*zNl || zNl==zEnd ); - //g_stderr("input: zNl=%d z=<<<%.*s>>>", (int)*zNl, (zNl-z), z); - unsigned char const * const zBOL = z; - cmpp__skip_space_c(&z, zNl); - if( z+nD < zNl && 0==memcmp(z, zD, nD) ){ - /* Found a directive delimiter. */ - if( zBOL!=z ){ - /* Do not emit space from the same line which preceeds a - delimiter */ - zLeft = z; - } - while( zNl>z && zNllineNo; - ++zNl; - nNl2 = strcspn((char const *)zNl, "\n"); - if( !nNl2 ) break; - nNlTotal += nNl2; - zNl += nNl2; - } - assert( zNl<=zEnd && "Else our input was not NUL-terminated"); - if( nNlTotal >= CmppArgs_BufSize ){ - fatal("Directive line is too long (%u)", - (unsigned)(zNl-z)); - break; - } - tflush; - t->token.zBegin = z + nD; - t->token.zEnd = zNl; - cmpp__skip_space_c(&t->token.zBegin, t->token.zEnd); - t->token.ttype = TT_Line; - t->token.lineNo = t->lineNo++; - t->zPos = t->token.zEnd + 1; - if( 0 ){ - g_stderr("token=<<%.*s>>", (t->token.zEnd - t->token.zBegin), - t->token.zBegin); - } - return 1; - } - z = zNl+1; - ++t->lineNo; - tflush; - //g_stderr("line #%d no match\n",(int)t->lineNo); - } - tflush; - t->zPos = z; - return 0; -#undef tflush -} - -void cmpp_kvp_parse(cmpp_kvp * p, char const *zKey, int nKey, - cmpp_kvp_op_e opPolicy){ - char chEq = 0; - char opLen = 0; - *p = cmpp_kvp_empty; - p->k.z = zKey; - p->k.n = cmpp_strlen(zKey, nKey); - switch( opPolicy ){ - case cmpp_kvp_op_none: break; - case cmpp_kvp_op_eq1: - chEq = '='; - opLen = 1; - break; - default: - assert(!"don't use these yet"); - /* todo: ==, !=, <=, <, >, >= */ - chEq = '='; - opLen = 1; - break; - } - assert( chEq ); - p->op = cmpp_kvp_op_none; - const char * const zEnd = p->k.z + p->k.n; - for(const char * zPos = p->k.z ; *zPos && zPosop = cmpp_kvp_op_eq1; - p->k.n = (unsigned)(zPos - zKey); - zPos += opLen; - assert( zPos <= zEnd ); - p->v.z = zPos; - p->v.n = (unsigned)(zEnd - zPos); - break; - } - } - cmpp_affirm_legal_key(p->k.z, p->k.n); -} - -static void cmpp_t_out_expand(CmppTokenizer * const t, - unsigned char const * zFrom, - unsigned int n){ - unsigned char const *zLeft = zFrom; - unsigned char const * const zEnd = zFrom + n; - unsigned char const *z = AT_OFF==g.flags.atPolicy ? zEnd : zLeft; - unsigned char const chEol = (unsigned char)'\n'; - int state = 0 /* 0==looking for opening @ - ** 1==looking for closing @ */; - if( 0 ){ - g_warn("zLeft=%d %c", (int)*zLeft, *zLeft); - } -#define tflush \ - if(z>zEnd) z=zEnd; \ - if(zLefttoken; - - assert(t->zBegin); - assert(t->zEnd > t->zBegin); - if(!t->zPos) t->zPos = t->zBegin; - t->args.pKw = 0; - t->args.argc = 0; - *tok = CmppToken_empty; - if( !CmppTokenizer__delim_search(t) ){ - return 0; - } - /* Split t->token into arguments for the line's keyword */ - int i, argc = 0, prevChar = 0; - const unsigned tokLen = (unsigned)(tok->zEnd - tok->zBegin); - unsigned char * zKwd; - unsigned char * zEsc; - unsigned char * zz; - - assert(TT_Line==tok->ttype); - g_debug(2,("token @ line %u len=%u [[[%.*s]]]\n", - tok->lineNo, tokLen, tokLen, tok->zBegin)); - zKwd = &t->args.lineBuf[0]; - memcpy(zKwd, tok->zBegin, tokLen); - memset(zKwd + tokLen, 0, sizeof(t->args.lineBuf) - tokLen); - for( zEsc = 0, zz = zKwd; *zz; ++zz ){ - /* Convert backslash-escaped newlines to whitespace */ - switch((int)*zz){ - case (int)'\\': - if(zEsc) zEsc = 0; - else zEsc = zz; - break; - case (int)'\n': - assert(zEsc && "Should not have an unescaped newline?"); - if(zEsc==zz-1){ - *zEsc = (unsigned char)' '; - /* FIXME?: memmove() lnBuf content one byte to the left here - ** to collapse backslash and newline into a single - ** byte. Also consider collapsing all leading space on the - ** next line. (Much later: or just collapse the output as we go, - ** effectively shrinking the line.) */ - } - zEsc = 0; - *zz = (unsigned char)' '; - break; - default: - zEsc = 0; - break; - } - } - t->args.argv[argc++] = zKwd; - for( zz = zKwd; *zz; ++zz ){ - if(isspace(*zz)){ - *zz = 0; - break; - } - } - t->args.pKw = CmppKeyword_search((char const *)zKwd); - if(!t->args.pKw){ - fatal("Unknown keyword '%s' at line %u\n", (char const *)zKwd, - tok->lineNo); - } - for( ++zz ; *zz && isspace(*zz); ++zz ){} - if(t->args.pKw->bTokenize){ - for( ; *zz; prevChar = *zz, ++zz ){ - /* Split string into word-shaped tokens. - ** TODO ?= quoted strings, for the sake of the - ** #error keyword. */ - if(isspace(*zz)){ - assert(zz!=zKwd && "Leading space was stripped earlier."); - *zz = 0; - }else{ - if(argc == (int)CmppArgs_Max){ - fatal("Too many arguments @ line %u: %.*s", - tok->lineNo, tokLen, tok->zBegin); - }else if(zz>zKwd && !prevChar){ - t->args.argv[argc++] = zz; - } - } - } - }else{ - /* Treat rest of line as one token */ - if(*zz) t->args.argv[argc++] = zz; - } - tok->ttype = t->args.pKw->ttype; - if(g.flags.doDebug>1){ - for(i = 0; i < argc; ++i){ - g_debug(0,("line %u arg #%d=%s\n", - tok->lineNo, i, - (char const *)t->args.argv[i])); - } - } - t->args.argc = argc; - return 1; -} - -/* Internal error reporting helper for cmpp_keyword_f() impls. */ -static CMPP_NORETURN void cmpp_kwd__err_(char const *zFile, int line, - CmppKeyword const * pKw, - CmppTokenizer const *t, - char const *zFmt, ...){ - va_list va; - g_stderr("%s @ %s line %u:", - pKw->zName, t->zName, t->token.lineNo); - va_start(va, zFmt); - g.tok = 0 /* stop fatalv__base() from duplicating the file info */; - fatalv__base(zFile, line, zFmt, va); - /* not reached */ - va_end(va); -} -#define cmpp_kwd__err(...) cmpp_kwd__err_(__FILE__,__LINE__, __VA_ARGS__) -#define cmpp_t__err(T,...) cmpp_kwd__err_(__FILE__,__LINE__, (T)->args.pKw, (T), __VA_ARGS__) - -/* No-op cmpp_keyword_f() impl. */ -static void cmpp_kwd_noop(CmppKeyword const * pKw, CmppTokenizer *t){ - (void)pKw; - (void)t; -} - -/* #error impl. */ -static void cmpp_kwd_error(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else{ - assert(t->args.argc < 3); - const char *zBegin = t->args.argc>1 - ? (const char *)t->args.argv[1] : 0; - cmpp_t__err(t, "%s", zBegin ? zBegin : "(no additional info)"); - } -} - -/* Impl. for #define, #undef */ -static void cmpp_kwd_define(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting one or more arguments"); - }else{ - int i = 1; - for( ; i < t->args.argc; ++i){ - char const * const zArg = (char const *)t->args.argv[i]; - cmpp_affirm_legal_key(zArg, -1); - if( TT_Define==pKw->ttype ){ - db_define_add( zArg, NULL ); - }else{ - db_define_rm( zArg ); - } - } - } -} - -static int cmpp_val_matches(char const *zGlob, char const *zRhs){ - return 0==sqlite3_strglob(zGlob, zRhs); -} - -typedef int (*cmpp_vcmp_f)(char const *zLhs, char const *zRhs); - -/* -** Accepts a key in the form X or X=Y. In the former case, it uses -** db_define_get_bool(kvp->k) to determine its truthiness, else it -** compares the kvp->v part to kvp->k's defined value to determine -** truthiness. -** -** Unless... -** -** If bCheckDefined is true is true then (A) it returns true if the -** value is defined and (B) fails fatally if given an X=Y-format key. -** -** Returns true if zKey evals to true, else false. -*/ -//static -int cmpp_kvp_truth(CmppKeyword const * const pKw, - CmppTokenizer const * const t, - cmpp_kvp const * const kvp, - int bCheckDefined){ - int buul = 0; - if( kvp->v.z ){ - if( bCheckDefined ){ - cmpp_kwd__err(pKw, t, "Value part is not legal for " - "is-defined checks: %.s", - kvp->k.n, kvp->k.z); - } - char * zVal = 0; - unsigned int nVal = 0; - buul = db_define_get(kvp->k.z, (int)kvp->k.n, &zVal, &nVal); - //g_debug(0,("checking key[%.*s]=%.*s\n", (zEq-zKey), zKey, nVal, zVal)); - if( kvp->v.n && nVal ){ - /* FIXME? do this with a query */ - /*g_debug(0,("if get-define [%.*s]=[%.*s] zValPart=%s\n", - (zEq-zKey), zKey, - nVal, zVal, zValPart));*/ - buul = cmpp_val_matches(kvp->v.z, zVal); - //g_debug(0,("buul=%d\n", buul)); - }else{ - assert( 0==kvp->v.n || 0==nVal ); - buul = kvp->v.n == nVal; - } - db_free(zVal); - }else{ - if( bCheckDefined ){ - buul = db_define_has(kvp->k.z, kvp->k.n); - }else{ - buul = db_define_get_bool(kvp->k.z, kvp->k.n); - } - } - return buul; -} - -#if 0 -/* -** A thin proxy for cmpp_kvp_truth(). -*/ -static int cmpp_key_truth(CmppKeyword const * pKw, - CmppTokenizer const * t, - char const *zKey, int bCheckDefined){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, cmpp_kvp_op_eq1); - return cmpp_kvp_truth(pKw, t, &kvp, bCheckDefined); -} -#endif - -//static -cmpp_kvp_op_e cmpp_t_is_op(CmppTokenizer const * t, int arg){ - if( t->args.argc > arg ){ - char const * const z = (char const *)t->args.argv[arg]; -#define E(N,S) if( strcmp(S,z) ) return cmpp_kvp_op_ ## N; else - cmpp_kvp_op_map(E) -#undef E - if(0) {} - } - return cmpp_kvp_op_none; -} - -/* -** A single part of an #if-type expression. They are parsed from -** CmppTokenizer::args in this form: -** -** not* defined{0,1} key[=[value]] -*/ -struct CmppExprDef { - /* The key part of the input. */ - cmpp_kvp kvp; - struct { - int ndx; - int next; - } arg; - CmppTokenizer const * tizer; - /* Set to 0 or 1 depending how many "not" are parsed. */ - unsigned char bNegated; - /* Set to 1 if "defined" is parsed. */ - unsigned char bCheckDefined; -}; -typedef struct CmppExprDef CmppExprDef; -#define CmppExprDef_empty_m {cmpp_kvp_empty_m,{0,0},0,0,0} -static const CmppExprDef CmppExprDef_empty = CmppExprDef_empty_m; - -/* -** Evaluate cep to true or false and return that value: -** -** If cep->bCheckDefined, return the result of db_define_has(). -** -** Else if cep->kvp.v.z is not NULL then fetch the define's value -** and return the result of cmpp_val_matches(cep->kvp.v.z,thatValue). -** -** Else return the result of db_define_get_bool(). -** -** The returned result accounts for cep->bNegated. -*/ -static int CmppExprDef_eval(CmppExprDef const * cep){ - int buul = 0; - - if( cep->bCheckDefined ){ - assert( !cep->kvp.v.n ); - buul = db_define_has(cep->kvp.k.z, (int)cep->kvp.k.n); - }else if( cep->kvp.v.z ){ - unsigned nVal = 0; - char * zVal = 0; - buul = db_define_get(cep->kvp.k.z, cep->kvp.k.n, &zVal, &nVal); - if( nVal ){ - buul = cmpp_val_matches(cep->kvp.v.z, zVal); - } - db_free(zVal); - }else{ - buul = db_define_get_bool(cep->kvp.k.z, cep->kvp.k.n); - } - return cep->bNegated ? !buul : buul; -} - -/* -** Expects t->args, starting at t->args.argv[startArg], to parse to -** one CmmpExprDef. It clears cep and repopulates it with info about -** the parse. Fails fatally on a parse error. -** -** Returns true if it reads one, false if it doesn't, and fails fatally -** if what it tries to parse is not empty but is not a CmppExprDef. -** -** Specifically, it parses: -** -** not+ defined? Word[=value] -** -*/ -static int CmppExprDef_read_one(CmppKeyword const * pKw, - CmppTokenizer const * t, - int startArg, CmppExprDef * cep){ - char const *zKey = 0; - *cep = CmppExprDef_empty; - cep->arg.ndx = startArg; - assert( t->args.pKw ); - assert( t->args.pKw==pKw ); - cep->tizer = t; - for(int i = startArg; !zKey && iargs.argc; ++i ){ - char const * z = (char const *)t->args.argv[i]; - if( 0==strcmp(z, "not") ){ - cep->bNegated = !cep->bNegated; - }else if( 0==strcmp(z,"defined") ){ - if( cep->bCheckDefined ){ - cmpp_kwd__err(pKw, t, - "Cannot use 'defined' more than once"); - } - cep->bCheckDefined = 1; - }else{ - assert( !zKey ); - cmpp_kvp_parse(&cep->kvp, z, -1, cmpp_kvp_op_eq1); - if( cep->bCheckDefined && cep->kvp.v.z ){ - cmpp_kwd__err(pKw, t, "Cannot use X=Y keys with 'defined'"); - cep->arg.next = ++i; - } - return 1; - } - } - return 0; -} - -/* -** Evals pStart and then proceeds to process any remaining arguments -** in t->args as RHS expressions. Returns the result of the expression -** as a bool. -** -** Specifically, it parses: -** -** and|or CmppExprDef -** -** Where CmppExprDef is the result of CmppExprDef_read_one(). -*/ -static int CmppExprDef_parse_cond(CmppKeyword const *pKw, - CmppTokenizer *t, - CmppExprDef const * pStart){ - enum { Op_none = 0, Op_And, Op_Or }; - int lhs = CmppExprDef_eval(pStart); - int op = Op_none; - int i = pStart->arg.next; - for( ; i < t->args.argc; ++i ){ - CmppExprDef eNext = CmppExprDef_empty; - char const *z = (char const *)t->args.argv[i]; - if( 0==strcmp("and",z) ){ - if( Op_none!=op ) goto multiple_ops; - op = Op_And; - continue; - }else if( 0==strcmp("or",z) ){ - if( Op_none!=op ) goto multiple_ops; - op = Op_Or; - continue; - }else if( !CmppExprDef_read_one(pKw, t, i, &eNext) ){ - if( Op_none!=op ){ - cmpp_t__err(t, "Stray operator: %s",z); - } - } - assert( eNext.kvp.k.z ); - int const rhs = CmppExprDef_eval(&eNext); - switch( op ){ - case Op_none: break; - case Op_And: lhs = lhs && rhs; break; - case Op_Or: lhs = lhs || rhs; break; - default: - assert(!"cannot happen"); - fatal("this cannot happen"); - } - op = Op_none; - } - if( Op_none!=op ){ - cmpp_t__err(t, "Extra operator at end of expression"); - }else if( i < t->args.argc ){ - assert(!"cannot happen"); - cmpp_kwd__err(t->args.pKw, t, "Unhandled extra arguments"); - }else{ - return lhs; - } - assert(!"not reached"); -multiple_ops: - cmpp_t__err(t,"Cannot have multiple operators"); - return 0 /* not reached */; -} - -/* Impl. for #if, #elif, #assert. */ -static void cmpp_kwd_if(CmppKeyword const * pKw, CmppTokenizer *t){ - CmppParseState tmpState = TS_Start; - CmppExprDef cep = CmppExprDef_empty; - //int buul = 0; - assert( TT_If==pKw->ttype - || TT_Elif==pKw->ttype - || TT_Assert==pKw->ttype); - if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting an argument"); - } - CmppExprDef_read_one(pKw, t, 1, &cep); - if( !cep.kvp.k.z ){ - cmpp_kwd__err(pKw, t, "Missing key argument"); - } - /*g_debug(0,("%s %s level %u pstate=%d bNot=%d bCheckDefined=%d\n", - pKw->zName, zKey, t->level.ndx, (int)CT_pstate(t), - bNot, bCheckDefined));*/ - switch(pKw->ttype){ - case TT_Assert: - break; - case TT_Elif: - switch(CT_pstate(t)){ - case TS_If: break; - case TS_IfPassed: CT_level(t).flags |= CmppLevel_F_ELIDE; return; - default: - cmpp_kwd__err(pKw, t, "'%s' used out of context", - pKw->zName); - } - break; - case TT_If: - CmppLevel_push(t); - break; - default: - assert(!"cannot happen"); - cmpp_kwd__err(pKw, t, "Unexpected keyword token type"); - break; - } - if( CmppExprDef_parse_cond( pKw, t, &cep ) ){ - CT_pstate(t) = tmpState = TS_IfPassed; - CT_skipLevel(t) = 0; - }else{ - if( TT_Assert==pKw->ttype ){ - cmpp_kwd__err(pKw, t, "Assertion failed: %s", - /* fixme: emit the whole line. We don't have it - handy in a readily-printable form. */ - cep.kvp.k.z); - } - CT_pstate(t) = TS_If /* also for TT_Elif */; - CT_skipLevel(t) = 1; - g_debug(3,("setting CT_skipLevel = 1 @ level %d\n", t->level.ndx)); - } - if( TT_If==pKw->ttype ){ - unsigned const lvlIf = t->level.ndx; - CmppToken const lvlToken = CT_level(t).token; - while(cmpp_next_keyword_line(t)){ - cmpp_process_keyword(t); - if(lvlIf > t->level.ndx){ - assert(TT_Endif == t->token.ttype); - break; - } -#if 0 - if(TS_IfPassed==tmpState){ - tmpState = TS_Start; - t->level.stack[lvlIf].flags |= CmppLevel_F_ELIDE; - g_debug(1,("Setting ELIDE for TS_IfPassed @ lv %d (lvlIf=%d)\n", t->level.ndx, lvlIf)); - } -#endif - } - if(lvlIf <= t->level.ndx){ - cmpp_kwd__err(pKw, t, - "Input ended inside an unterminated %sif " - "opened at [%s] line %u", - g.delim.z, t->zName, lvlToken.lineNo); - } - } -} - -/* Impl. for #else. */ -static void cmpp_kwd_else(CmppKeyword const * pKw, CmppTokenizer *t){ - if(t->args.argc>1){ - cmpp_kwd__err(pKw, t, "Expecting no arguments"); - } - switch(CT_pstate(t)){ - case TS_IfPassed: CT_skipLevel(t) = 1; break; - case TS_If: CT_skipLevel(t) = 0; break; - default: - cmpp_kwd__err(pKw, t, "'%s' with no matching 'if'", - pKw->zName); - } - /*g_debug(0,("else flags=0x%02x skipLevel=%u\n", - CT_level(t).flags, CT_level(t).skipLevel));*/ - CT_pstate(t) = TS_Else; -} - -/* Impl. for #endif. */ -static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ - /* Maintenance reminder: we ignore all arguments after the endif - ** to allow for constructs like: - ** - ** #endif // foo - ** - ** in a manner which does not require a specific comment style */ - switch(CT_pstate(t)){ - case TS_Else: - case TS_If: - case TS_IfPassed: - break; - default: - cmpp_kwd__err(pKw, t, "'%s' with no matching 'if'", - pKw->zName); - } - CmppLevel_pop(t); -} - -/* Impl. for #include. */ -static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ - char const * zFile; - char * zResolved; - int bRaw = 0 /* -raw flag */; - if(CT_skip(t)) return; - else if(t->args.argc<2 || t->args.argc>3){ - cmpp_kwd__err(pKw, t, "Expected args: ?-raw? filename"); - } - if(t->args.argc==2){ - zFile = (const char *)t->args.argv[1]; - }else{ - if( 0==strcmp("-raw", (char*)t->args.argv[1]) ){ - bRaw = 1; - }else{ - cmpp_kwd__err(pKw, t, "Unhandled argument: %s", - t->args.argv[1]); - } - zFile = (const char *)t->args.argv[2]; - } - if(!bRaw && db_including_has(zFile)){ - /* Note that different spellings of the same filename - ** will elude this check, but that seems okay, as different - ** spellings means that we're not re-running the exact same - ** invocation. We might want some other form of multi-include - ** protection, rather than this, however. There may well be - ** sensible uses for recursion. */ - cmpp_t__err(t, "Recursive include of file: %s", zFile); - } - zResolved = db_include_search(zFile); - if(zResolved){ - if( bRaw ) db_including_add(zFile, t->zName, t->token.lineNo); - cmpp_process_file(zResolved, bRaw); - if( bRaw ) db_include_rm(zFile); - db_free(zResolved); - }else{ - cmpp_t__err(t, "file not found: %s", zFile); - } -} - -static void cmpp_dump_defines( FILE * fp, int bIndent ){ - sqlite3_stmt * const q = g_stmt(GStmt_defSelAll); - while( SQLITE_ROW==sqlite3_step(q) ){ - unsigned char const * zK = sqlite3_column_text(q, 0); - unsigned char const * zV = sqlite3_column_text(q, 1); - int const nK = sqlite3_column_bytes(q, 0); - int const nV = sqlite3_column_bytes(q, 1); - fprintf(fp, "%s%.*s = %.*s\n", - bIndent ? "\t" : "", nK, zK, nV, zV); - } - g_stmt_reset(q); -} - -/* Impl. for #pragma. */ -static void cmpp_kwd_pragma(CmppKeyword const * pKw, CmppTokenizer *t){ - const char * zArg; - if(CT_skip(t)) return; - else if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting an argument"); - } - zArg = (const char *)t->args.argv[1]; -#define M(X) 0==strcmp(zArg,X) - if(M("defines")){ - cmpp_dump_defines(stderr, 1); - } - else if(M("chomp-F")){ - g.flags.chompF = 1; - }else if(M("no-chomp-F")){ - g.flags.chompF = 0; - } -#if 0 - /* now done by cmpp_kwd_at_policy() */ - else if(M("@")){ - if(t->args.argc>2){ - g.flags.atPolicy = - AtPolicy_fromStr((char const *)t->args.argv[2], 1); - }else{ - g.flags.atPolicy = AT_DEFAULT; - } - }else if(M("no-@")){ - g.flags.atPolicy = AT_OFF; - } -#endif - else{ - cmpp_kwd__err(pKw, t, "Unknown pragma: %s", zArg); - } -#undef M -} - -static void db_step_reset(sqlite3_stmt * const q, char const * zErrTip){ - db_affirm_rc(sqlite3_step(q), zErrTip); - g_stmt_reset(q); -} - -static void cmpp_sp_begin(CmppTokenizer * const t){ - db_step_reset(g_stmt(GStmt_spBegin), "Starting savepoint"); - ++t->nSavepoint; -} - -static void cmpp_sp_rollback(CmppTokenizer * const t){ - if( !t->nSavepoint ){ - cmpp_t__err(t, "Cannot roll back: no active savepoint"); - } - db_step_reset(g_stmt(GStmt_spRollback), - "Rolling back savepoint"); - db_step_reset(g_stmt(GStmt_spRelease), - "Releasing rolled-back savepoint"); - --t->nSavepoint; -} - -static void cmpp_sp_commit(CmppTokenizer * const t){ - if( !t->nSavepoint ){ - cmpp_t__err(t, "Cannot commit: no active savepoint"); - } - db_step_reset(g_stmt(GStmt_spRelease), "Rolling back savepoint"); - --t->nSavepoint; -} - -void CmppTokenizer_cleanup(CmppTokenizer * const t){ - while( t->nSavepoint ){ - cmpp_sp_rollback(t); - } -} - -/* Impl. for #savepoint. */ -static void cmpp_kwd_savepoint(CmppKeyword const * pKw, CmppTokenizer *t){ - const char * zArg; - if(CT_skip(t)) return; - else if(t->args.argc!=2){ - cmpp_kwd__err(pKw, t, "Expecting one argument"); - } - zArg = (const char *)t->args.argv[1]; -#define M(X) 0==strcmp(zArg,X) - if(M("begin")){ - cmpp_sp_begin(t); - }else if(M("rollback")){ - cmpp_sp_rollback(t); - }else if(M("commit")){ - cmpp_sp_commit(t); - }else{ - cmpp_kwd__err(pKw, t, "Unknown savepoint option: %s", zArg); - } -#undef SP_NAME -#undef M -} - -/* #stder impl. */ -static void cmpp_kwd_stderr(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else{ - const char *zBegin = t->args.argc>1 - ? (const char *)t->args.argv[1] : 0; - if(zBegin){ - g_stderr("%s:%u: %s\n", t->zName, t->token.lineNo, zBegin); - }else{ - g_stderr("%s:%u: (no %.*s%s argument)\n", - t->zName, t->token.lineNo, - g.delim.n, g.delim.z, pKw->zName); - } - } -} - -/* Impl. for the @ policy. */ -static void cmpp_kwd_at_policy(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else if(t->args.argc<2){ - g.flags.atPolicy = AT_DEFAULT; - }else{ - g.flags.atPolicy = AtPolicy_fromStr((char const*)t->args.argv[1], 1); - } -} - - -#if 0 -/* Impl. for dummy placeholder. */ -static void cmpp_kwd_todo(CmppKeyword const * pKw, CmppTokenizer *t){ - (void)t; - g_debug(0,("TODO: keyword handler for %s\n", pKw->zName)); -} -#endif - -CmppKeyword aKeywords[] = { -/* Keep these sorted by zName */ -#define S(NAME) DStrings.NAME.z, DStrings.NAME.n - {S(Comment), 0, TT_Comment, cmpp_kwd_noop}, - {S(AtPolicy), 1, TT_AtPolicy, cmpp_kwd_at_policy}, - {S(Assert),1, TT_Assert, cmpp_kwd_if}, - {S(Define), 1, TT_Define, cmpp_kwd_define}, - {S(Elif), 1, TT_Elif, cmpp_kwd_if}, - {S(Else), 1, TT_Else, cmpp_kwd_else}, - {S(Endif), 0, TT_Endif, cmpp_kwd_endif}, - {S(Error), 0, TT_Error, cmpp_kwd_error}, - {S(If), 1, TT_If, cmpp_kwd_if}, - {S(Include), 1, TT_Include, cmpp_kwd_include}, - {S(Pragma), 1, TT_Pragma, cmpp_kwd_pragma}, - {S(Savepoint), 1, TT_Savepoint, cmpp_kwd_savepoint}, - {S(Stderr), 0, TT_Stderr, cmpp_kwd_stderr}, - {S(Undef), 1, TT_Undef, cmpp_kwd_define}, -#undef S - {0,0,TT_Invalid, 0} -}; - -static int cmpp_CmppKeyword(const void *p1, const void *p2){ - char const * zName = (const char *)p1; - CmppKeyword const * kw = (CmppKeyword const *)p2; - return strcmp(zName, kw->zName); -} - -CmppKeyword const * CmppKeyword_search(const char *zName){ - return (CmppKeyword const *)bsearch(zName, &aKeywords[0], - sizeof(aKeywords)/sizeof(aKeywords[0]) - 1, - sizeof(aKeywords[0]), - cmpp_CmppKeyword); -} - -void cmpp_process_keyword(CmppTokenizer * const t){ - assert(t->args.pKw); - assert(t->args.argc); - t->args.pKw->xCall(t->args.pKw, t); - t->args.pKw = 0; - t->args.argc = 0; -} - -void cmpp_process_string(const char * zName, - unsigned char const * zIn, - int nIn){ - nIn = cmpp__strlenu(zIn, nIn); - if( !nIn ) return; - CmppTokenizer const * const oldTok = g.tok; - CmppTokenizer ct = CmppTokenizer_empty; - ct.zName = zName; - ct.zBegin = zIn; - ct.zEnd = zIn + nIn; - while(cmpp_next_keyword_line(&ct)){ - cmpp_process_keyword(&ct); - } - if(0!=ct.level.ndx){ - CmppLevel const * const lv = CmppLevel_get(&ct); - fatal("Input ended inside an unterminated nested construct " - "opened at [%s] line %u", zName, lv->token.lineNo); - } - CmppTokenizer_cleanup(&ct); - g.tok = oldTok; -} - -void cmpp_process_file(const char * zName, int bRaw){ - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, zName, "r"); - g_FileWrapper_link(&fw); - FileWrapper_slurp(&fw); - g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); - if( fw.zContent ){ - if( bRaw ){ - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, zName, "rb"); - g_FileWrapper_link(&fw); - FileWrapper_slurp(&fw); - if( 1!=fwrite(fw.zContent, fw.nContent, 1, g.out.pFile) ){ - fatal("fwrite() failed with errno %d\n", errno); - } - g_FileWrapper_close(&fw); - }else{ - cmpp_process_string(zName, fw.zContent, fw.nContent); - } - } - g_FileWrapper_close(&fw); -} - - -void fatalv__base(char const *zFile, int line, - char const *zFmt, va_list va){ - FILE * const fp = stderr; - fflush(stdout); - fputc('\n', fp); - if( g.flags.doDebug ){ - fprintf(fp, "%s: ", g.zArgv0); - if( zFile ){ - fprintf(fp, "%s:%d ",zFile, line); - } - } - if( g.tok ){ - fprintf(fp,"@%s:%d: ", - (g.tok->zName && 0==strcmp("-",g.tok->zName)) - ? "" - : g.tok->zName, - g.tok->lineNo); - } - if(zFmt && *zFmt){ - vfprintf(fp, zFmt, va); - } - fputc('\n', fp); - fflush(fp); - exit(1); -} - -void fatal__base(char const *zFile, int line, - char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - fatalv__base(zFile, line, zFmt, va); - va_end(va); -} - -#undef CT_level -#undef CT_pstate -#undef CT_skipLevel -#undef CT_skip -#undef CLvl_skip - -static void usage(int isErr){ - FILE * const fOut = isErr ? stderr : stdout; - fprintf(fOut, "Usage: %s [flags] [infile...]\n", g.zArgv0); - fprintf(fOut, - "Flags and filenames may be in any order and " - "they are processed in that order.\n" - "\nFlags:\n"); -#define GAP " " -#define arg(F,D) fprintf(fOut,"\n %s\n" GAP "%s\n",F, D) - arg("-o|--outfile FILE","Send output to FILE (default=- (stdout)).\n" - GAP "Because arguments are processed in order, this should\n" - GAP "normally be given before -f."); - arg("-f|--file FILE","Process FILE (default=- (stdin)).\n" - GAP "All non-flag arguments are assumed to be the input files."); - arg("-DXYZ[=value]","Define XYZ to the given value (default=1)."); - arg("-UXYZ","Undefine all defines matching glob XYZ."); - arg("-IXYZ","Add dir XYZ to the " CMPP_DEFAULT_DELIM "include path."); - arg("-FXYZ=filename", - "Define XYZ to the raw contents of the given file.\n" - GAP "The file is not processed as by " CMPP_DEFAULT_DELIM"include\n" - GAP "Maybe it should be. Or maybe we need a new flag for that."); - arg("-d|--delimiter VALUE", "Set keyword delimiter to VALUE " - "(default=" CMPP_DEFAULT_DELIM ")."); - arg("--@policy retain|elide|error|off", - "Specifies how to handle @tokens@ (default=off).\n" - GAP "off = do not look for @tokens@\n" - GAP "retain = parse @tokens@ and retain any undefined ones\n" - GAP "elide = parse @tokens@ and elide any undefined ones\n" - GAP "error = parse @tokens@ and error out for any undefined ones" - ); - arg("-@", "Equivalent to --@policy=error."); - arg("-no-@", "Equivalent to --@policy=off (the default)."); - arg("--sql-trace", "Send a trace of all SQL to stderr."); - arg("--sql-trace-x", - "Like --sql-trace but expand all bound values in the SQL."); - arg("--no-sql-trace", "Disable SQL tracing (default)."); - arg("--chomp-F", "One trailing newline is trimmed from files " - "read via -FXYZ=filename."); - arg("--no-chomp-F", "Disable --chomp-F (default)."); -#undef arg -#undef GAP - fputs("\nFlags which require a value accept either " - "--flag=value or --flag value.\n\n",fOut); -} - -/* -** Expects that *ndx points to the current argv entry and that it is a -** flag which expects a value. This function checks for --flag=val and -** (--flag val) forms. If a value is found then *ndx is adjusted (if -** needed) to point to the next argument after the value and *zVal is -** pointed to the value. If no value is found then it fails fatally. -*/ -static void get_flag_val(int argc, char const * const * argv, int * ndx, - char const **zVal){ - char const * zEq = strchr(argv[*ndx], '='); - if( zEq ){ - *zVal = zEq+1; - return; - } - if(*ndx+1>=argc){ - fatal("Missing value for flag '%s'", argv[*ndx]); - } - *zVal = argv[++*ndx]; -} - -static int arg_is_flag( char const *zFlag, char const *zArg, - char const **zValIfEqX ){ - *zValIfEqX = 0; - if( 0==strcmp(zFlag, zArg) ) return 1; - char const * z = strchr(zArg,'='); - if( z && z>zArg ){ - /* compare the part before the '=' */ - if( 0==strncmp(zFlag, zArg, z-zArg) ){ - if( !zFlag[z-zArg] ){ - *zValIfEqX = z+1; - return 1; - } - /* Else it was a prefix match. */ - } - } - return 0; -} - -static void define_argv(int argc, char const * const * argv){ - sqlite3_str * const s = sqlite3_str_new(g.db); - sqlite3_str_append(s, "c-pp::argv=", 11); - for( int i = 0; i < argc; ++i ){ - if( i ) sqlite3_str_appendchar(s, 1, ' '); - sqlite3_str_appendf(s, "%s", argv[i]); - } - char * const z = sqlite3_str_finish(s); - assert(z); - if(z){ - db_define_add(z, NULL); - sqlite3_free(z); - } -} - -int main(int argc, char const * const * argv){ - int rc = 0; - int inclCount = 0; - int nFile = 0; - int ndxTrace = 0; - int expandMode = 0; - char const * zVal = 0; -#define ARGVAL if( !zVal ) get_flag_val(argc, argv, &i, &zVal) -#define M(X) arg_is_flag(X, zArg, &zVal) -#define ISFLAG(X) else if(M(X)) -#define ISFLAG2(X,Y) else if(M(X) || M(Y)) -#define NOVAL if( zVal ) fatal("Unexpected value for %s", zArg) -#define g_out_open \ - if(!g.out.pFile) FileWrapper_open(&g.out, "-", "w"); \ - if(!inclCount){ db_include_dir_add("."); ++inclCount; } (void)0 - - g.zArgv0 = argv[0]; -#define DOIT if(doIt) - for(int doIt = 0; doIt<2; ++doIt){ - /** - Loop through the flags twice. The first time we just validate - and look for --help/-?. The second time we process the flags. - This approach allows us to easily chain multiple files and - flags: - - ./c-pp -Dfoo -o foo x.y -Ufoo -Dbar -o bar x.y - */ - DOIT{ - atexit(cmpp_atexit); - if( 1==ndxTrace ){ - /* Ensure that we start with tracing in the early stage if - --sql-trace is the first arg, in order to log schema - setup. */ - g.sqlTrace.pFile = stderr; - g.sqlTrace.expandSql = expandMode; - } - cmpp_initdb(); - define_argv(argc, argv); - } - for(int i = 1; i < argc; ++i){ - int negate = 0; - char const * zArg = argv[i]; - //g_stderr("i=%d zArg=%s\n", i, zArg); - zVal = 0; - while('-'==*zArg) ++zArg; - if(zArg==argv[i]/*not a flag*/){ - zVal = zArg; - goto do_infile; - } - if( 0==strncmp(zArg,"no-",3) ){ - zArg += 3; - negate = 1; - } - ISFLAG2("?","help"){ - NOVAL; - usage(0); - goto end; - }else if('D'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -D"); - DOIT { - db_define_add(zArg, 0); - } - }else if('F'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -F"); - DOIT { - db_define_add_file(zArg); - } - }else if('U'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -U"); - DOIT { - db_define_rm(zArg); - } - }else if('I'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing directory for -I"); - DOIT { - db_include_dir_add(zArg); - ++inclCount; - } - } - ISFLAG2("o","outfile"){ - ARGVAL; - DOIT { - FileWrapper_open(&g.out, zVal, "w"); - } - } - ISFLAG2("f","file"){ - ARGVAL; - do_infile: - DOIT { - ++nFile; - g_out_open; - cmpp_process_file(zVal, 0); - } - } - ISFLAG("e"){ - ARGVAL; - DOIT { - ++nFile; - g_out_open; - cmpp_process_string("-e script", ustr_c(zVal), -1); - } - } - ISFLAG("@"){ - NOVAL; - DOIT { - assert( AT_DEFAULT!=AT_OFF ); - g.flags.atPolicy = negate ? AT_OFF : AT_DEFAULT; - } - } - ISFLAG("@policy"){ - AtPolicy aup; - ARGVAL; - aup = AtPolicy_fromStr(zVal, 1); - DOIT { - g.flags.atPolicy = aup; - } - } - ISFLAG("debug"){ - NOVAL; - g.flags.doDebug += negate ? -1 : 1; - } - ISFLAG("sql-trace"){ - NOVAL; - /* Needs to be set before the start of the second pass, when - the db is inited. */ - g.sqlTrace.expandSql = 0; - DOIT { - g.sqlTrace.pFile = negate ? (FILE*)0 : stderr; - }else if( !ndxTrace && !negate ){ - ndxTrace = i; - expandMode = 0; - } - } - ISFLAG("sql-trace-x"){ - NOVAL; - g.sqlTrace.expandSql = 1; - DOIT { - g.sqlTrace.pFile = negate ? (FILE*)0 : stderr; - }else if( !ndxTrace && !negate ){ - ndxTrace = i; - expandMode = 1; - } - } - ISFLAG("chomp-F"){ - NOVAL; - DOIT g.flags.chompF = !negate; - } - ISFLAG2("d","delimiter"){ - ARGVAL; - if( !doIt ){ - g.delim.z = zVal; - g.delim.n = (unsigned short)strlen(zVal); - if(!g.delim.n) fatal("Keyword delimiter may not be empty."); - } - } - ISFLAG2("dd", "dump-defines"){ - DOIT { - FILE * const fp = stderr; - fprintf(fp, "All %sdefine entries:\n", g.delim.z); - cmpp_dump_defines(fp, 1); - } - } - else{ - fatal("Unhandled flag: %s", argv[i]); - } - } - DOIT { - if(!nFile){ - if(!g.out.zName) g.out.zName = "-"; - if(!inclCount){ - db_include_dir_add("."); - ++inclCount; - } - FileWrapper_open(&g.out, g.out.zName, "w"); - cmpp_process_file("-", 0); - } - } - } - end: - g_cleanup(1); - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/ext/wasm/demo-worker1-promiser.c-pp.html b/ext/wasm/demo-worker1-promiser.c-pp.html index a1005beb93..c54b46aadb 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.html +++ b/ext/wasm/demo-worker1-promiser.c-pp.html @@ -10,7 +10,7 @@ Worker1-promiser (ESM) tests //#else Worker1-promiser tests -//#endif +//#/if
worker-promise tests
@@ -37,6 +37,6 @@ //#else -//#endif +//#/if diff --git a/ext/wasm/demo-worker1-promiser.c-pp.js b/ext/wasm/demo-worker1-promiser.c-pp.js index c129e21281..1521edfc17 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.js +++ b/ext/wasm/demo-worker1-promiser.c-pp.js @@ -19,7 +19,7 @@ import {default as promiserFactory} from "./jswasm/sqlite3-worker1-promiser.mjs" "use strict"; const promiserFactory = globalThis.sqlite3Worker1Promiser.v2; delete globalThis.sqlite3Worker1Promiser; -//#endif +//#/if (async function(){ const T = globalThis.SqliteTestUtil; const eOutput = document.querySelector('#test-output'); @@ -53,7 +53,7 @@ delete globalThis.sqlite3Worker1Promiser; before workerPromise is set. */ console.warn("This is the v2 interface - you don't need an onready() function."); }, -//#endif +//#/if debug: 1 ? undefined : (...args)=>console.debug('worker debug',...args), onunhandled: function(ev){ error("Unhandled worker message:",ev.data); diff --git a/ext/wasm/fiddle/index.c-pp.html b/ext/wasm/fiddle/index.c-pp.html index 1f818286b5..7fe9a12a77 100644 --- a/ext/wasm/fiddle/index.c-pp.html +++ b/ext/wasm/fiddle/index.c-pp.html @@ -27,7 +27,7 @@ --> -//#endif +//#/if