/* DO NOT EDIT! ** This file is automatically generated by the script in the canonical ** SQLite source tree at tool/mkshellc.tcl. That script combines source ** code from various constituent source files of SQLite into this single ** "shell.c" file used to implement the SQLite command-line shell. ** ** Most of the code found below comes from the "src/shell.c.in" file in ** the canonical SQLite source tree. That main file contains "INCLUDE" ** lines that specify other files in the canonical source tree that are ** inserted to getnerate this complete program source file. ** ** The code from multiple files is combined into this single "shell.c" ** source file to help make the command-line program easier to compile. ** ** To modify this program, get a copy of the canonical SQLite source tree, ** edit the src/shell.c.in" and/or some of the other files that are included ** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. */ /* ** 2001 September 15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** This file contains code to implement the "sqlite" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) /* This needs to come before any includes for MSVC compiler */ #define _CRT_SECURE_NO_WARNINGS #endif u32; u16; /* ** Optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include ** file. Note that this macro has a like effect on sqlite3.c compilation. */ #define SHELL_STRINGIFY_(f) … #define SHELL_STRINGIFY(f) … #ifdef SQLITE_CUSTOM_INCLUDE # include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) #endif /* ** Determine if we are dealing with WinRT, which provides only a subset of ** the full Win32 API. */ #if !defined(SQLITE_OS_WINRT) #define SQLITE_OS_WINRT … #endif /* ** If SQLITE_SHELL_FIDDLE is defined then the shell is modified ** somewhat for use as a WASM module in a web browser. This flag ** should only be used when building the "fiddle" web application, as ** the browser-mode build has much different user input requirements ** and this build mode rewires the user input subsystem to account for ** that. */ /* ** Warning pragmas copied from msvc.h in the core. */ #if defined(_MSC_VER) #pragma warning(disable : 4054) #pragma warning(disable : 4055) #pragma warning(disable : 4100) #pragma warning(disable : 4127) #pragma warning(disable : 4130) #pragma warning(disable : 4152) #pragma warning(disable : 4189) #pragma warning(disable : 4206) #pragma warning(disable : 4210) #pragma warning(disable : 4232) #pragma warning(disable : 4244) #pragma warning(disable : 4305) #pragma warning(disable : 4306) #pragma warning(disable : 4702) #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ /* ** No support for loadable extensions in VxWorks. */ #if (defined(__RTP__) || defined(_WRS_KERNEL)) && !SQLITE_OMIT_LOAD_EXTENSION #define SQLITE_OMIT_LOAD_EXTENSION … #endif /* ** Enable large-file support for fopen() and friends on unix. */ #ifndef SQLITE_DISABLE_LFS #define _LARGE_FILE … # ifndef _FILE_OFFSET_BITS #define _FILE_OFFSET_BITS … # endif #define _LARGEFILE_SOURCE … #endif #if defined(SQLITE_SHELL_FIDDLE) && !defined(_POSIX_SOURCE) /* ** emcc requires _POSIX_SOURCE (or one of several similar defines) ** to expose strdup(). */ #define _POSIX_SOURCE #endif #include <stdlib.h> #include <string.h> #include <stdio.h> #include <assert.h> #include <math.h> #include "sqlite3.h" i64; u64; u8; #if SQLITE_USER_AUTHENTICATION # include "sqlite3userauth.h" #endif #include <ctype.h> #include <stdarg.h> #if !defined(_WIN32) && !defined(WIN32) # include <signal.h> # if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) # include <pwd.h> # endif #endif #if (!defined(_WIN32) && !defined(WIN32)) || defined(__MINGW32__) # include <unistd.h> # include <dirent.h> #define GETPID … # if defined(__MINGW32__) #define DIRENT … # ifndef S_ISLNK #define S_ISLNK … # endif # endif #else #define GETPID … #endif #include <sys/types.h> #include <sys/stat.h> #if HAVE_READLINE # include <readline/readline.h> # include <readline/history.h> #endif #if HAVE_EDITLINE # include <editline/readline.h> #endif #if HAVE_EDITLINE || HAVE_READLINE #define shell_add_history … #define shell_read_history … #define shell_write_history … #define shell_stifle_history … #define shell_readline … #elif HAVE_LINENOISE # include "linenoise.h" #define shell_add_history … #define shell_read_history … #define shell_write_history … #define shell_stifle_history … #define shell_readline … #else #define shell_read_history(X) … #define shell_write_history(X) … #define shell_stifle_history(X) … #define SHELL_USE_LOCAL_GETLINE … #endif #ifndef deliberate_fall_through /* Quiet some compilers about some of our intentional code. */ # if defined(GCC_VERSION) && GCC_VERSION>=7000000 #define deliberate_fall_through … # else #define deliberate_fall_through # endif #endif #if defined(_WIN32) || defined(WIN32) # if SQLITE_OS_WINRT #define SQLITE_OMIT_POPEN … # else # include <io.h> # include <fcntl.h> #define isatty … # ifndef access #define access … # endif # ifndef unlink #define unlink … # endif # ifndef strdup #define strdup … # endif # undef popen #define popen … # undef pclose #define pclose … # endif #else /* Make sure isatty() has a prototype. */ extern int isatty(int); # if !defined(__RTP__) && !defined(_WRS_KERNEL) && !defined(SQLITE_WASI) /* popen and pclose are not C89 functions and so are ** sometimes omitted from the <stdio.h> header */ extern FILE *popen(const char*,const char*); extern int pclose(FILE*); # else #define SQLITE_OMIT_POPEN … # endif #endif #if defined(_WIN32_WCE) /* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() * thus we always assume that we have a console. That can be * overridden with the -batch command line option. */ #define isatty … #endif /* ctype macros that work with signed characters */ #define IsSpace(X) … #define IsDigit(X) … #define ToLower(X) … #if defined(_WIN32) || defined(WIN32) #if SQLITE_OS_WINRT #include <intrin.h> #endif #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include <windows.h> /* string conversion routines only needed on Win32 */ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE … #ifdef SQLITE_SHELL_FIDDLE /* Deselect most features from the console I/O package for Fiddle. */ #define SQLITE_CIO_NO_REDIRECT #define SQLITE_CIO_NO_CLASSIFY #define SQLITE_CIO_NO_TRANSLATE #define SQLITE_CIO_NO_SETMODE #endif /************************* Begin ../ext/consio/console_io.h ******************/ /* ** 2023 November 1 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ******************************************************************************** ** This file exposes various interfaces used for console and other I/O ** by the SQLite project command-line tools. These interfaces are used ** at either source conglomeration time, compilation time, or run time. ** This source provides for either inclusion into conglomerated, ** "single-source" forms or separate compilation then linking. ** ** Platform dependencies are "hidden" here by various stratagems so ** that, provided certain conditions are met, the programs using this ** source or object code compiled from it need no explicit conditional ** compilation in their source for their console and stream I/O. ** ** The symbols and functionality exposed here are not a public API. ** This code may change in tandem with other project code as needed. ** ** When this .h file and its companion .c are directly incorporated into ** a source conglomeration (such as shell.c), the preprocessor symbol ** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O ** translation for Windows is effected for the build. */ #define HAVE_CONSOLE_IO_H … #ifndef SQLITE_INTERNAL_LINKAGE #define SQLITE_INTERNAL_LINKAGE … # include <stdio.h> #else #define SHELL_NO_SYSINC … #endif #ifndef SQLITE3_H /* # include "sqlite3.h" */ #endif #ifndef SQLITE_CIO_NO_CLASSIFY /* Define enum for use with following function. */ StreamsAreConsole; /* ** Classify the three standard I/O streams according to whether ** they are connected to a console attached to the process. ** ** Returns the bit-wise OR of SAC_{In,Out,Err}Console values, ** or SAC_NoConsole if none of the streams reaches a console. ** ** This function should be called before any I/O is done with ** the given streams. As a side-effect, the given inputs are ** recorded so that later I/O operations on them may be done ** differently than the C library FILE* I/O would be done, ** iff the stream is used for the I/O functions that follow, ** and to support the ones that use an implicit stream. ** ** On some platforms, stream or console mode alteration (aka ** "Setup") may be made which is undone by consoleRestore(). */ SQLITE_INTERNAL_LINKAGE StreamsAreConsole consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ); /* A usual call for convenience: */ #define SQLITE_STD_CONSOLE_INIT() … /* ** After an initial call to consoleClassifySetup(...), renew ** the same setup it effected. (A call not after is an error.) ** This will restore state altered by consoleRestore(); ** ** Applications which run an inferior (child) process which ** inherits the same I/O streams may call this function after ** such a process exits to guard against console mode changes. */ SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void); /* ** Undo any side-effects left by consoleClassifySetup(...). ** ** This should be called after consoleClassifySetup() and ** before the process terminates normally. It is suitable ** for use with the atexit() C library procedure. After ** this call, no console I/O should be done until one of ** console{Classify or Renew}Setup(...) is called again. ** ** Applications which run an inferior (child) process that ** inherits the same I/O streams might call this procedure ** before so that said process will have a console setup ** however users have configured it or come to expect. */ SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ); #else /* defined(SQLITE_CIO_NO_CLASSIFY) */ #define consoleClassifySetup … #define consoleRenewSetup … #define consoleRestore … #endif /* defined(SQLITE_CIO_NO_CLASSIFY) */ #ifndef SQLITE_CIO_NO_REDIRECT /* ** Set stream to be used for the functions below which write ** to "the designated X stream", where X is Output or Error. ** Returns the previous value. ** ** Alternatively, pass the special value, invalidFileStream, ** to get the designated stream value without setting it. ** ** Before the designated streams are set, they default to ** those passed to consoleClassifySetup(...), and before ** that is called they default to stdout and stderr. ** ** It is error to close a stream so designated, then, without ** designating another, use the corresponding {o,e}Emit(...). */ SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream; SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf); # ifdef CONSIO_SET_ERROR_STREAM SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf); # endif #else #define setOutputStream … #define setErrorStream … #endif /* !defined(SQLITE_CIO_NO_REDIRECT) */ #ifndef SQLITE_CIO_NO_TRANSLATE /* ** Emit output like fprintf(). If the output is going to the ** console and translation from UTF-8 is necessary, perform ** the needed translation. Otherwise, write formatted output ** to the provided stream almost as-is, possibly with newline ** translation as specified by set{Binary,Text}Mode(). */ SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...); /* Like fPrintfUtf8 except stream is always the designated output. */ SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...); /* Like fPrintfUtf8 except stream is always the designated error. */ SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...); /* ** Emit output like fputs(). If the output is going to the ** console and translation from UTF-8 is necessary, perform ** the needed translation. Otherwise, write given text to the ** provided stream almost as-is, possibly with newline ** translation as specified by set{Binary,Text}Mode(). */ SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO); /* Like fPutsUtf8 except stream is always the designated output. */ SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z); /* Like fPutsUtf8 except stream is always the designated error. */ SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z); /* ** Emit output like fPutsUtf8(), except that the length of the ** accepted char or character sequence is limited by nAccept. ** ** Returns the number of accepted char values. */ #ifdef CONSIO_SPUTB SQLITE_INTERNAL_LINKAGE int fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept); /* Like fPutbUtf8 except stream is always the designated output. */ #endif SQLITE_INTERNAL_LINKAGE int oPutbUtf8(const char *cBuf, int nAccept); /* Like fPutbUtf8 except stream is always the designated error. */ #ifdef CONSIO_EPUTB SQLITE_INTERNAL_LINKAGE int ePutbUtf8(const char *cBuf, int nAccept); #endif /* ** Collect input like fgets(...) with special provisions for input ** from the console on platforms that require same. Defers to the ** C library fgets() when input is not from the console. Newline ** translation may be done as set by set{Binary,Text}Mode(). As a ** convenience, pfIn==NULL is treated as stdin. */ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn); /* Like fGetsUtf8 except stream is always the designated input. */ /* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */ #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ #ifndef SQLITE_CIO_NO_SETMODE /* ** Set given stream for binary mode, where newline translation is ** not done, or for text mode where, for some platforms, newlines ** are translated to the platform's conventional char sequence. ** If bFlush true, flush the stream. ** ** An additional side-effect is that if the stream is one passed ** to consoleClassifySetup() as an output, it is flushed first. ** ** Note that binary/text mode has no effect on console I/O ** translation. On all platforms, newline to the console starts ** a new line and CR,LF chars from the console become a newline. */ SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush); SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush); #endif #ifdef SQLITE_CIO_PROMPTED_IN typedef struct Prompts { int numPrompts; const char **azPrompts; } Prompts; /* ** Macros for use of a line editor. ** ** The following macros define operations involving use of a ** line-editing library or simple console interaction. ** A "T" argument is a text (char *) buffer or filename. ** A "N" argument is an integer. ** ** SHELL_ADD_HISTORY(T) // Record text as line(s) of history. ** SHELL_READ_HISTORY(T) // Read history from file named by T. ** SHELL_WRITE_HISTORY(T) // Write history to file named by T. ** SHELL_STIFLE_HISTORY(N) // Limit history to N entries. ** ** A console program which does interactive console input is ** expected to call: ** SHELL_READ_HISTORY(T) before collecting such input; ** SHELL_ADD_HISTORY(T) as record-worthy input is taken; ** SHELL_STIFLE_HISTORY(N) after console input ceases; then ** SHELL_WRITE_HISTORY(T) before the program exits. */ /* ** Retrieve a single line of input text from an input stream. ** ** If pfIn is the input stream passed to consoleClassifySetup(), ** and azPrompt is not NULL, then a prompt is issued before the ** line is collected, as selected by the isContinuation flag. ** Array azPrompt[{0,1}] holds the {main,continuation} prompt. ** ** If zBufPrior is not NULL then it is a buffer from a prior ** call to this routine that can be reused, or will be freed. ** ** The result is stored in space obtained from malloc() and ** must either be freed by the caller or else passed back to ** this function as zBufPrior for reuse. ** ** This function may call upon services of a line-editing ** library to interactively collect line edited input. */ SQLITE_INTERNAL_LINKAGE char * shellGetLine(FILE *pfIn, char *zBufPrior, int nLen, short isContinuation, Prompts azPrompt); #endif /* defined(SQLITE_CIO_PROMPTED_IN) */ /* ** TBD: Define an interface for application(s) to generate ** completion candidates for use by the line-editor. ** ** This may be premature; the CLI is the only application ** that does this. Yet, getting line-editing melded into ** console I/O is desirable because a line-editing library ** may have to establish console operating mode, possibly ** in a way that interferes with the above functionality. */ #if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE)) /* Skip over as much z[] input char sequence as is valid UTF-8, ** limited per nAccept char's or whole characters and containing ** no char cn such that ((1<<cn) & ccm)!=0. On return, the ** sequence z:return (inclusive:exclusive) is validated UTF-8. ** Limit: nAccept>=0 => char count, nAccept<0 => character */ SQLITE_INTERNAL_LINKAGE const char* zSkipValidUtf8(const char *z, int nAccept, long ccm); #endif /************************* End ../ext/consio/console_io.h ********************/ /************************* Begin ../ext/consio/console_io.c ******************/ /* ** 2023 November 4 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ******************************************************************************** ** This file implements various interfaces used for console and stream I/O ** by the SQLite project command-line tools, as explained in console_io.h . ** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there. */ #ifndef SQLITE_CDECL #define SQLITE_CDECL #endif #ifndef SHELL_NO_SYSINC # include <stdarg.h> # include <string.h> # include <stdlib.h> # include <limits.h> # include <assert.h> /* # include "sqlite3.h" */ #endif #ifndef HAVE_CONSOLE_IO_H # include "console_io.h" #endif #if defined(_MSC_VER) # pragma warning(disable : 4204) #endif #ifndef SQLITE_CIO_NO_TRANSLATE # if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT # ifndef SHELL_NO_SYSINC # include <io.h> # include <fcntl.h> # undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN # include <windows.h> # endif #define CIO_WIN_WC_XLATE … # else # ifndef SHELL_NO_SYSINC # include <unistd.h> # endif #define CIO_WIN_WC_XLATE … # endif #else #define CIO_WIN_WC_XLATE … #endif #if CIO_WIN_WC_XLATE /* Character used to represent a known-incomplete UTF-8 char group (�) */ static WCHAR cBadGroup = 0xfffd; #endif #if CIO_WIN_WC_XLATE static HANDLE handleOfFile(FILE *pf){ int fileDesc = _fileno(pf); union { intptr_t osfh; HANDLE fh; } fid = { (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE }; return fid.fh; } #endif #ifndef SQLITE_CIO_NO_TRANSLATE PerStreamTags; /* Define NULL-like value for things which can validly be 0. */ #define SHELL_INVALID_FILE_PTR … # if CIO_WIN_WC_XLATE #define SHELL_INVALID_CONS_MODE … # endif # if CIO_WIN_WC_XLATE #define PST_INITIALIZER … # else #define PST_INITIALIZER … # endif /* Quickly say whether a known output is going to the console. */ # if CIO_WIN_WC_XLATE static short pstReachesConsole(PerStreamTags *ppst){ return (ppst->hx != INVALID_HANDLE_VALUE); } # else #define pstReachesConsole(ppst) … # endif # if CIO_WIN_WC_XLATE static void restoreConsoleArb(PerStreamTags *ppst){ if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode); } # else #define restoreConsoleArb(ppst) … # endif /* Say whether FILE* appears to be a console, collect associated info. */ static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){ … } # ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING #define ENABLE_VIRTUAL_TERMINAL_PROCESSING … # endif # if CIO_WIN_WC_XLATE /* Define console modes for use with the Windows Console API. */ #define SHELL_CONI_MODE … #define SHELL_CONO_MODE … # endif ConsoleInfo; static short isValidStreamInfo(PerStreamTags *ppst){ … } static ConsoleInfo consoleInfo = …; SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = …; # if CIO_WIN_WC_XLATE static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){ if( pstReachesConsole(ppst) ){ DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE; SetConsoleMode(ppst->hx, cm); } } # else #define maybeSetupAsConsole(ppst,odir) … # endif SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){ … } SQLITE_INTERNAL_LINKAGE StreamsAreConsole consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){ … } SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){ … } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ #ifdef SQLITE_CIO_INPUT_REDIR /* Say whether given FILE* is among those known, via either ** consoleClassifySetup() or set{Output,Error}Stream, as ** readable, and return an associated PerStreamTags pointer ** if so. Otherwise, return 0. */ static PerStreamTags * isKnownReadable(FILE *pf){ static PerStreamTags *apst[] = { &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0 }; int ix = 0; do { if( apst[ix]->pf == pf ) break; } while( apst[++ix] != 0 ); return apst[ix]; } #endif #ifndef SQLITE_CIO_NO_TRANSLATE /* Say whether given FILE* is among those known, via either ** consoleClassifySetup() or set{Output,Error}Stream, as ** writable, and return an associated PerStreamTags pointer ** if so. Otherwise, return 0. */ static PerStreamTags * isKnownWritable(FILE *pf){ … } static FILE *designateEmitStream(FILE *pf, unsigned chix){ … } SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){ … } # ifdef CONSIO_SET_ERROR_STREAM SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){ return designateEmitStream(pf, 2); } # endif #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ #ifndef SQLITE_CIO_NO_SETMODE # if CIO_WIN_WC_XLATE static void setModeFlushQ(FILE *pf, short bFlush, int mode){ if( bFlush ) fflush(pf); _setmode(_fileno(pf), mode); } # else #define setModeFlushQ … # endif SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){ … } SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){ … } # undef setModeFlushQ #else /* defined(SQLITE_CIO_NO_SETMODE) */ #define setBinaryMode … #define setTextMode … #endif /* defined(SQLITE_CIO_NO_SETMODE) */ #ifndef SQLITE_CIO_NO_TRANSLATE # if CIO_WIN_WC_XLATE /* Write buffer cBuf as output to stream known to reach console, ** limited to ncTake char's. Return ncTake on success, else 0. */ static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){ int rv = 0; if( z!=NULL ){ int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0); if( nwc > 0 ){ WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR)); if( zw!=NULL ){ nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc); if( nwc > 0 ){ /* Translation from UTF-8 to UTF-16, then WCHARs out. */ if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){ rv = ncTake; } } sqlite3_free(zw); } } } return rv; } /* For {f,o,e}PrintfUtf8() when stream is known to reach console. */ static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){ char *z = sqlite3_vmprintf(zFormat, ap); if( z ){ int rv = conZstrEmit(ppst, z, (int)strlen(z)); sqlite3_free(z); return rv; }else return 0; } # endif /* CIO_WIN_WC_XLATE */ # ifdef CONSIO_GET_EMIT_STREAM static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix, PerStreamTags *ppst){ PerStreamTags *rv = isKnownWritable(pf); short isValid = (rv!=0)? isValidStreamInfo(rv) : 0; if( rv != 0 && isValid ) return rv; streamOfConsole(pf, ppst); return ppst; } # endif /* Get stream info, either for designated output or error stream when ** chix equals 1 or 2, or for an arbitrary stream when chix == 0. ** In either case, ppst references a caller-owned PerStreamTags ** struct which may be filled in if none of the known writable ** streams is being held by consoleInfo. The ppf parameter is a ** byref output when chix!=0 and a byref input when chix==0. */ static PerStreamTags * getEmitStreamInfo(unsigned chix, PerStreamTags *ppst, /* in/out */ FILE **ppf){ … } SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){ … } SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){ … } SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){ … } SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){ … } SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){ … } SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){ … } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ #if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE)) /* Skip over as much z[] input char sequence as is valid UTF-8, ** limited per nAccept char's or whole characters and containing ** no char cn such that ((1<<cn) & ccm)!=0. On return, the ** sequence z:return (inclusive:exclusive) is validated UTF-8. ** Limit: nAccept>=0 => char count, nAccept<0 => character */ SQLITE_INTERNAL_LINKAGE const char* zSkipValidUtf8(const char *z, int nAccept, long ccm){ … } #endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/ #ifndef SQLITE_CIO_NO_TRANSLATE # ifdef CONSIO_SPUTB SQLITE_INTERNAL_LINKAGE int fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){ assert(pfO!=0); # if CIO_WIN_WC_XLATE PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO); if( pstReachesConsole(ppst) ){ int rv; maybeSetupAsConsole(ppst, 1); rv = conZstrEmit(ppst, cBuf, nAccept); if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst); return rv; }else { # endif return (int)fwrite(cBuf, 1, nAccept, pfO); # if CIO_WIN_WC_XLATE } # endif } # endif SQLITE_INTERNAL_LINKAGE int oPutbUtf8(const char *cBuf, int nAccept){ … } # ifdef CONSIO_EPUTB SQLITE_INTERNAL_LINKAGE int ePutbUtf8(const char *cBuf, int nAccept){ FILE *pfErr; PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */ PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr); # if CIO_WIN_WC_XLATE if( pstReachesConsole(ppst) ){ return conZstrEmit(ppst, cBuf, nAccept); }else { # endif return (int)fwrite(cBuf, 1, nAccept, pfErr); # if CIO_WIN_WC_XLATE } # endif } # endif /* defined(CONSIO_EPUTB) */ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){ … } #endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */ #if defined(_MSC_VER) # pragma warning(default : 4204) #endif #undef SHELL_INVALID_FILE_PTR /************************* End ../ext/consio/console_io.c ********************/ #ifndef SQLITE_SHELL_FIDDLE /* From here onward, fgets() is redirected to the console_io library. */ #define fgets(b,n,f) … /* * Define macros for emitting output text in various ways: * sputz(s, z) => emit 0-terminated string z to given stream s * sputf(s, f, ...) => emit varargs per format f to given stream s * oputz(z) => emit 0-terminated string z to default stream * oputf(f, ...) => emit varargs per format f to default stream * eputz(z) => emit 0-terminated string z to error stream * eputf(f, ...) => emit varargs per format f to error stream * oputb(b, n) => emit char buffer b[0..n-1] to default stream * * Note that the default stream is whatever has been last set via: * setOutputStream(FILE *pf) * This is normally the stream that CLI normal output goes to. * For the stand-alone CLI, it is stdout with no .output redirect. * * The ?putz(z) forms are required for the Fiddle builds for string literal * output, in aid of enforcing format string to argument correspondence. */ #define sputz(s,z) … #define sputf … #define oputz(z) … #define oputf … #define eputz(z) … #define eputf … #define oputb(buf,na) … #else /* For Fiddle, all console handling and emit redirection is omitted. */ /* These next 3 macros are for emitting formatted output. When complaints * from the WASM build are issued for non-formatted output, (when a mere * string literal is to be emitted, the ?putz(z) forms should be used. * (This permits compile-time checking of format string / argument mismatch.) */ #define oputf … #define eputf … #define sputf … /* These next 3 macros are for emitting simple string literals. */ #define oputz … #define eputz … #define sputz … #define oputb … #endif /* True if the timer is enabled */ static int enableTimer = …; /* A version of strcmp() that works with NULL values */ static int cli_strcmp(const char *a, const char *b){ … } static int cli_strncmp(const char *a, const char *b, size_t n){ … } /* Return the current wall-clock time */ static sqlite3_int64 timeOfDay(void){ … } #if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) #include <sys/time.h> #include <sys/resource.h> /* VxWorks does not support getrusage() as far as we can determine */ #if defined(_WRS_KERNEL) || defined(__RTP__) struct rusage { struct timeval ru_utime; /* user CPU time used */ struct timeval ru_stime; /* system CPU time used */ }; #define getrusage … #endif /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; /* CPU time at start */ static sqlite3_int64 iBegin; /* Wall-clock time at start */ /* ** Begin timing an operation */ static void beginTimer(void){ … } /* Return the difference of two time_structs in seconds */ static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ … } /* ** Print the timing results. */ static void endTimer(void){ … } #define BEGIN_TIMER … #define END_TIMER … #define HAS_TIMER … #elif (defined(_WIN32) || defined(WIN32)) /* Saved resource information for the beginning of an operation */ static HANDLE hProcess; static FILETIME ftKernelBegin; static FILETIME ftUserBegin; static sqlite3_int64 ftWallBegin; typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, LPFILETIME, LPFILETIME); static GETPROCTIMES getProcessTimesAddr = NULL; /* ** Check to see if we have timer support. Return 1 if necessary ** support found (or found previously). */ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { #if !SQLITE_OS_WINRT /* GetProcessTimes() isn't supported in WIN95 and some other Windows ** versions. See if the version we are running on has it, and if it ** does, save off a pointer to it and the current process handle. */ hProcess = GetCurrentProcess(); if( hProcess ){ HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); if( NULL != hinstLib ){ getProcessTimesAddr = (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); if( NULL != getProcessTimesAddr ){ return 1; } FreeLibrary(hinstLib); } } #endif } return 0; } /* ** Begin timing an operation */ static void beginTimer(void){ if( enableTimer && getProcessTimesAddr ){ FILETIME ftCreation, ftExit; getProcessTimesAddr(hProcess,&ftCreation,&ftExit, &ftKernelBegin,&ftUserBegin); ftWallBegin = timeOfDay(); } } /* Return the difference of two FILETIME structs in seconds */ static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ sqlite_int64 i64Start = *((sqlite_int64 *) pStart); sqlite_int64 i64End = *((sqlite_int64 *) pEnd); return (double) ((i64End - i64Start) / 10000000.0); } /* ** Print the timing results. */ static void endTimer(void){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); sputf(stdout, "Run Time: real %.3f user %f sys %f\n", (ftWallEnd - ftWallBegin)*0.001, timeDiff(&ftUserBegin, &ftUserEnd), timeDiff(&ftKernelBegin, &ftKernelEnd)); } } #define BEGIN_TIMER … #define END_TIMER … #define HAS_TIMER … #else #define BEGIN_TIMER #define END_TIMER #define HAS_TIMER … #endif /* ** Used to prevent warnings about unused parameters */ #define UNUSED_PARAMETER(x) … /* ** Number of elements in an array */ #define ArraySize(X) … /* ** If the following flag is set, then command execution stops ** at an error if we are not interactive. */ static int bail_on_error = …; /* ** Treat stdin as an interactive input if the following variable ** is true. Otherwise, assume stdin is connected to a file or pipe. */ static int stdin_is_interactive = …; /* ** On Windows systems we need to know if standard output is a console ** in order to show that UTF-16 translation is done in the sign-on ** banner. The following variable is true if it is the console. */ static int stdout_is_console = …; /* ** The following is the open SQLite database. We make a pointer ** to this database a static variable so that it can be accessed ** by the SIGINT handler to interrupt database processing. */ static sqlite3 *globalDb = …; /* ** True if an interrupt (Control-C) has been received. */ static volatile int seenInterrupt = …; /* ** This is the name of our program. It is set in main(), used ** in a number of other places, mostly for error messages. */ static char *Argv0; /* ** Prompt strings. Initialized in main. Settable with ** .prompt main continue */ #define PROMPT_LEN_MAX … /* First line prompt. default: "sqlite> " */ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ static char continuePrompt[PROMPT_LEN_MAX]; /* This is variant of the standard-library strncpy() routine with the ** one change that the destination string is always zero-terminated, even ** if there is no zero-terminator in the first n-1 characters of the source ** string. */ static char *shell_strncpy(char *dest, const char *src, size_t n){ … } /* ** Optionally disable dynamic continuation prompt. ** Unless disabled, the continuation prompt shows open SQL lexemes if any, ** or open parentheses level if non-zero, or continuation prompt as set. ** This facility interacts with the scanner and process_input() where the ** below 5 macros are used. */ #ifdef SQLITE_OMIT_DYNAPROMPT #define CONTINUATION_PROMPT … #define CONTINUE_PROMPT_RESET #define CONTINUE_PROMPT_AWAITS … #define CONTINUE_PROMPT_AWAITC … #define CONTINUE_PAREN_INCR … #define CONTINUE_PROMPT_PSTATE … typedef void *t_NoDynaPrompt; #define SCAN_TRACKER_REFTYPE … #else #define CONTINUATION_PROMPT … #define CONTINUE_PROMPT_RESET … #define CONTINUE_PROMPT_AWAITS(p,s) … #define CONTINUE_PROMPT_AWAITC(p,c) … #define CONTINUE_PAREN_INCR(p,n) … #define CONTINUE_PROMPT_PSTATE … t_DynaPromptRef; #define SCAN_TRACKER_REFTYPE … static struct DynaPrompt { … } dynPrompt = …; /* Record parenthesis nesting level change, or force level to 0. */ static void trackParenLevel(struct DynaPrompt *p, int ni){ … } /* Record that a lexeme is opened, or closed with args==0. */ static void setLexemeOpen(struct DynaPrompt *p, char *s, char c){ … } /* Upon demand, derive the continuation prompt to display. */ static char *dynamicContinuePrompt(void){ … } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ … } /* Check a pointer to see if it is NULL. If it is NULL, exit with an ** out-of-memory error. */ static void shell_check_oom(const void *p){ … } /* ** Write I/O traces to the following stream. */ #ifdef SQLITE_ENABLE_IOTRACE static FILE *iotrace = 0; #endif /* ** This routine works like printf in that its first argument is a ** format string and subsequent arguments are values to be substituted ** in place of % fields. The result of formatting this string ** is written to iotrace. */ #ifdef SQLITE_ENABLE_IOTRACE static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_list ap; char *z; if( iotrace==0 ) return; va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); sputf(iotrace, "%s", z); sqlite3_free(z); } #endif /* ** Output string zUtf to Out stream as w characters. If w is negative, ** then right-justify the text. W is the width in UTF-8 characters, not ** in bytes. This is different from the %*.*s specification in printf ** since with %*.*s the width is measured in bytes, not characters. */ static void utf8_width_print(int w, const char *zUtf){ … } /* ** Determines if a string is a number of not. */ static int isNumber(const char *z, int *realnum){ … } /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. */ static int strlen30(const char *z){ … } /* ** Return the length of a string in characters. Multibyte UTF8 characters ** count as a single character. */ static int strlenChar(const char *z){ … } /* ** Return open FILE * if zFile exists, can be opened for read ** and is an ordinary file or a character stream source. ** Otherwise return 0. */ static FILE * openChrSource(const char *zFile){ … } /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer ** to the text. NULL is returned at end of file, or if malloc() ** fails. ** ** If zLine is not NULL then it is a malloced buffer returned from ** a previous call to this routine that may be reused. */ static char *local_getline(char *zLine, FILE *in){ … } /* ** Retrieve a single line of input text. ** ** If in==0 then read from standard input and prompt before each line. ** If isContinuation is true, then a continuation prompt is appropriate. ** If isContinuation is zero, then the main prompt should be used. ** ** If zPrior is not NULL then it is a buffer from a prior call to this ** routine that can be reused. ** ** The result is stored in space obtained from malloc() and must either ** be freed by the caller or else passed back into this routine via the ** zPrior argument for reuse. */ #ifndef SQLITE_SHELL_FIDDLE static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ … } #endif /* !SQLITE_SHELL_FIDDLE */ /* ** Return the value of a hexadecimal digit. Return -1 if the input ** is not a hex digit. */ static int hexDigitValue(char c){ … } /* ** Interpret zArg as an integer value, possibly with suffixes. */ static sqlite3_int64 integerValue(const char *zArg){ … } /* ** A variable length string to which one can append text. */ ShellText; struct ShellText { … }; /* ** Initialize and destroy a ShellText object */ static void initText(ShellText *p){ … } static void freeText(ShellText *p){ … } /* zIn is either a pointer to a NULL-terminated string in memory obtained ** from malloc(), or a NULL pointer. The string pointed to by zAppend is ** added to zIn, and the result returned in memory obtained from malloc(). ** zIn, if it was not NULL, is freed. ** ** If the third argument, quote, is not '\0', then it is used as a ** quote character for zAppend. */ static void appendText(ShellText *p, const char *zAppend, char quote){ … } /* ** Attempt to determine if identifier zName needs to be quoted, either ** because it contains non-alphanumeric characters, or because it is an ** SQLite keyword. Be conservative in this estimate: When in doubt assume ** that quoting is required. ** ** Return '"' if quoting is required. Return 0 if no quoting is required. */ static char quoteChar(const char *zName){ … } /* ** Construct a fake object name and column list to describe the structure ** of the view, virtual table, or table valued function zSchema.zName. */ static char *shellFakeSchema( sqlite3 *db, /* The database connection containing the vtab */ const char *zSchema, /* Schema of the database holding the vtab */ const char *zName /* The name of the virtual table */ ){ … } /* ** SQL function: strtod(X) ** ** Use the C-library strtod() function to convert string X into a double. ** Used for comparing the accuracy of SQLite's internal text-to-float conversion ** routines against the C-library. */ static void shellStrtod( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** SQL function: dtostr(X) ** ** Use the C-library printf() function to convert real value X into a string. ** Used for comparing the accuracy of SQLite's internal float-to-text conversion ** routines against the C-library. */ static void shellDtostr( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** SQL function: shell_module_schema(X) ** ** Return a fake schema for the table-valued function or eponymous virtual ** table X. */ static void shellModuleSchema( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** SQL function: shell_add_schema(S,X) ** ** Add the schema name X to the CREATE statement in S and return the result. ** Examples: ** ** CREATE TABLE t1(x) -> CREATE TABLE xyz.t1(x); ** ** Also works on ** ** CREATE INDEX ** CREATE UNIQUE INDEX ** CREATE VIEW ** CREATE TRIGGER ** CREATE VIRTUAL TABLE ** ** This UDF is used by the .schema command to insert the schema name of ** attached databases into the middle of the sqlite_schema.sql field. */ static void shellAddSchemaName( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** The source code for several run-time loadable extensions is inserted ** below by the ../tool/mkshellc.tcl script. Before processing that included ** code, we need to override some macros to make the included program code ** work here in the middle of this regular program. */ #define SQLITE_EXTENSION_INIT1 #define SQLITE_EXTENSION_INIT2(X) … #if defined(_WIN32) && defined(_MSC_VER) /************************* Begin test_windirent.h ******************/ /* ** 2015 November 30 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** This file contains declarations for most of the opendir() family of ** POSIX functions on Win32 using the MSVCRT. */ #if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) #define SQLITE_WINDIRENT_H /* ** We need several data types from the Windows SDK header. */ #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include "windows.h" /* ** We need several support functions from the SQLite core. */ /* #include "sqlite3.h" */ /* ** We need several things from the ANSI and MSVCRT headers. */ #include <stdio.h> #include <stdlib.h> #include <errno.h> #include <io.h> #include <limits.h> #include <sys/types.h> #include <sys/stat.h> /* ** We may need several defines that should have been in "sys/stat.h". */ #ifndef S_ISREG #define S_ISREG … #endif #ifndef S_ISDIR #define S_ISDIR … #endif #ifndef S_ISLNK #define S_ISLNK … #endif /* ** We may need to provide the "mode_t" type. */ #ifndef MODE_T_DEFINED #define MODE_T_DEFINED typedef unsigned short mode_t; #endif /* ** We may need to provide the "ino_t" type. */ #ifndef INO_T_DEFINED #define INO_T_DEFINED typedef unsigned short ino_t; #endif /* ** We need to define "NAME_MAX" if it was not present in "limits.h". */ #ifndef NAME_MAX # ifdef FILENAME_MAX #define NAME_MAX … # else #define NAME_MAX … # endif #endif /* ** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". */ #ifndef NULL_INTPTR_T #define NULL_INTPTR_T … #endif #ifndef BAD_INTPTR_T #define BAD_INTPTR_T … #endif /* ** We need to provide the necessary structures and related types. */ #ifndef DIRENT_DEFINED #define DIRENT_DEFINED typedef struct DIRENT DIRENT; typedef DIRENT *LPDIRENT; struct DIRENT { ino_t d_ino; /* Sequence number, do not use. */ unsigned d_attributes; /* Win32 file attributes. */ char d_name[NAME_MAX + 1]; /* Name within the directory. */ }; #endif #ifndef DIR_DEFINED #define DIR_DEFINED typedef struct DIR DIR; typedef DIR *LPDIR; struct DIR { intptr_t d_handle; /* Value returned by "_findfirst". */ DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ DIRENT d_next; /* DIRENT constructed based on "_findnext". */ }; #endif /* ** Provide a macro, for use by the implementation, to determine if a ** particular directory entry should be skipped over when searching for ** the next directory entry that should be returned by the readdir() or ** readdir_r() functions. */ #ifndef is_filtered #define is_filtered … #endif /* ** Provide the function prototype for the POSIX compatible getenv() ** function. This function is not thread-safe. */ extern const char *windirent_getenv(const char *name); /* ** Finally, we can provide the function prototypes for the opendir(), ** readdir(), readdir_r(), and closedir() POSIX functions. */ extern LPDIR opendir(const char *dirname); extern LPDIRENT readdir(LPDIR dirp); extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); extern INT closedir(LPDIR dirp); #endif /* defined(WIN32) && defined(_MSC_VER) */ /************************* End test_windirent.h ********************/ /************************* Begin test_windirent.c ******************/ /* ** 2015 November 30 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** This file contains code to implement most of the opendir() family of ** POSIX functions on Win32 using the MSVCRT. */ #if defined(_WIN32) && defined(_MSC_VER) /* #include "test_windirent.h" */ /* ** Implementation of the POSIX getenv() function using the Win32 API. ** This function is not thread-safe. */ const char *windirent_getenv( const char *name ){ static char value[32768]; /* Maximum length, per MSDN */ DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ memset(value, 0, sizeof(value)); dwRet = GetEnvironmentVariableA(name, value, dwSize); if( dwRet==0 || dwRet>dwSize ){ /* ** The function call to GetEnvironmentVariableA() failed -OR- ** the buffer is not large enough. Either way, return NULL. */ return 0; }else{ /* ** The function call to GetEnvironmentVariableA() succeeded ** -AND- the buffer contains the entire value. */ return value; } } /* ** Implementation of the POSIX opendir() function using the MSVCRT. */ LPDIR opendir( const char *dirname ){ struct _finddata_t data; LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); if( dirp==NULL ) return NULL; memset(dirp, 0, sizeof(DIR)); /* TODO: Remove this if Unix-style root paths are not used. */ if( sqlite3_stricmp(dirname, "/")==0 ){ dirname = windirent_getenv("SystemDrive"); } memset(&data, 0, sizeof(struct _finddata_t)); _snprintf(data.name, namesize, "%s\\*", dirname); dirp->d_handle = _findfirst(data.name, &data); if( dirp->d_handle==BAD_INTPTR_T ){ closedir(dirp); return NULL; } /* TODO: Remove this block to allow hidden and/or system files. */ if( is_filtered(data) ){ next: memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ closedir(dirp); return NULL; } /* TODO: Remove this block to allow hidden and/or system files. */ if( is_filtered(data) ) goto next; } dirp->d_first.d_attributes = data.attrib; strncpy(dirp->d_first.d_name, data.name, NAME_MAX); dirp->d_first.d_name[NAME_MAX] = '\0'; return dirp; } /* ** Implementation of the POSIX readdir() function using the MSVCRT. */ LPDIRENT readdir( LPDIR dirp ){ struct _finddata_t data; if( dirp==NULL ) return NULL; if( dirp->d_first.d_ino==0 ){ dirp->d_first.d_ino++; dirp->d_next.d_ino++; return &dirp->d_first; } next: memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; /* TODO: Remove this block to allow hidden and/or system files. */ if( is_filtered(data) ) goto next; dirp->d_next.d_ino++; dirp->d_next.d_attributes = data.attrib; strncpy(dirp->d_next.d_name, data.name, NAME_MAX); dirp->d_next.d_name[NAME_MAX] = '\0'; return &dirp->d_next; } /* ** Implementation of the POSIX readdir_r() function using the MSVCRT. */ INT readdir_r( LPDIR dirp, LPDIRENT entry, LPDIRENT *result ){ struct _finddata_t data; if( dirp==NULL ) return EBADF; if( dirp->d_first.d_ino==0 ){ dirp->d_first.d_ino++; dirp->d_next.d_ino++; entry->d_ino = dirp->d_first.d_ino; entry->d_attributes = dirp->d_first.d_attributes; strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); entry->d_name[NAME_MAX] = '\0'; *result = entry; return 0; } next: memset(&data, 0, sizeof(struct _finddata_t)); if( _findnext(dirp->d_handle, &data)==-1 ){ *result = NULL; return ENOENT; } /* TODO: Remove this block to allow hidden and/or system files. */ if( is_filtered(data) ) goto next; entry->d_ino = (ino_t)-1; /* not available */ entry->d_attributes = data.attrib; strncpy(entry->d_name, data.name, NAME_MAX); entry->d_name[NAME_MAX] = '\0'; *result = entry; return 0; } /* ** Implementation of the POSIX closedir() function using the MSVCRT. */ INT closedir( LPDIR dirp ){ INT result = 0; if( dirp==NULL ) return EINVAL; if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ result = _findclose(dirp->d_handle); } sqlite3_free(dirp); return result; } #endif /* defined(WIN32) && defined(_MSC_VER) */ /************************* End test_windirent.c ********************/ #define dirent … #endif /************************* Begin ../ext/misc/memtrace.c ******************/ /* ** 2019-01-21 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file implements an extension that uses the SQLITE_CONFIG_MALLOC ** mechanism to add a tracing layer on top of SQLite. If this extension ** is registered prior to sqlite3_initialize(), it will cause all memory ** allocation activities to be logged on standard output, or to some other ** FILE specified by the initializer. ** ** This file needs to be compiled into the application that uses it. ** ** This extension is used to implement the --memtrace option of the ** command-line shell. */ #include <assert.h> #include <string.h> #include <stdio.h> /* The original memory allocation routines */ static sqlite3_mem_methods memtraceBase; static FILE *memtraceOut; /* Methods that trace memory allocations */ static void *memtraceMalloc(int n){ … } static void memtraceFree(void *p){ … } static void *memtraceRealloc(void *p, int n){ … } static int memtraceSize(void *p){ … } static int memtraceRoundup(int n){ … } static int memtraceInit(void *p){ … } static void memtraceShutdown(void *p){ … } /* The substitute memory allocator */ static sqlite3_mem_methods ersaztMethods = …; /* Begin tracing memory allocations to out. */ int sqlite3MemTraceActivate(FILE *out){ … } /* Deactivate memory tracing */ int sqlite3MemTraceDeactivate(void){ … } /************************* End ../ext/misc/memtrace.c ********************/ /************************* Begin ../ext/misc/pcachetrace.c ******************/ /* ** 2023-06-21 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 ** mechanism to add a tracing layer on top of pluggable page cache of ** SQLite. If this extension is registered prior to sqlite3_initialize(), ** it will cause all page cache activities to be logged on standard output, ** or to some other FILE specified by the initializer. ** ** This file needs to be compiled into the application that uses it. ** ** This extension is used to implement the --pcachetrace option of the ** command-line shell. */ #include <assert.h> #include <string.h> #include <stdio.h> /* The original page cache routines */ static sqlite3_pcache_methods2 pcacheBase; static FILE *pcachetraceOut; /* Methods that trace pcache activity */ static int pcachetraceInit(void *pArg){ … } static void pcachetraceShutdown(void *pArg){ … } static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ … } static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ … } static int pcachetracePagecount(sqlite3_pcache *p){ … } static sqlite3_pcache_page *pcachetraceFetch( sqlite3_pcache *p, unsigned key, int crFg ){ … } static void pcachetraceUnpin( sqlite3_pcache *p, sqlite3_pcache_page *pPg, int bDiscard ){ … } static void pcachetraceRekey( sqlite3_pcache *p, sqlite3_pcache_page *pPg, unsigned oldKey, unsigned newKey ){ … } static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ … } static void pcachetraceDestroy(sqlite3_pcache *p){ … } static void pcachetraceShrink(sqlite3_pcache *p){ … } /* The substitute pcache methods */ static sqlite3_pcache_methods2 ersaztPcacheMethods = …; /* Begin tracing memory allocations to out. */ int sqlite3PcacheTraceActivate(FILE *out){ … } /* Deactivate memory tracing */ int sqlite3PcacheTraceDeactivate(void){ … } /************************* End ../ext/misc/pcachetrace.c ********************/ /************************* Begin ../ext/misc/shathree.c ******************/ /* ** 2017-03-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This SQLite extension implements functions that compute SHA3 hashes ** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. ** Two SQL functions are implemented: ** ** sha3(X,SIZE) ** sha3_query(Y,SIZE) ** ** The sha3(X) function computes the SHA3 hash of the input X, or NULL if ** X is NULL. ** ** The sha3_query(Y) function evaluates all queries in the SQL statements of Y ** and returns a hash of their results. ** ** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm ** is used. If SIZE is included it must be one of the integers 224, 256, ** 384, or 512, to determine SHA3 hash variant that is computed. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> #include <stdarg.h> #ifndef SQLITE_AMALGAMATION /* typedef sqlite3_uint64 u64; */ #endif /* SQLITE_AMALGAMATION */ /****************************************************************************** ** The Hash Engine */ /* ** Macros to determine whether the machine is big or little endian, ** and whether or not that determination is run-time or compile-time. ** ** For best performance, an attempt is made to guess at the byte-order ** using C-preprocessor macros. If that is unsuccessful, or if ** -DSHA3_BYTEORDER=0 is set, then byte-order is determined ** at run-time. */ #ifndef SHA3_BYTEORDER # if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ defined(__arm__) #define SHA3_BYTEORDER … # elif defined(sparc) || defined(__ppc__) #define SHA3_BYTEORDER … # else #define SHA3_BYTEORDER … # endif #endif /* ** State structure for a SHA3 hash in progress */ SHA3Context; struct SHA3Context { … }; /* ** A single step of the Keccak mixing function for a 1600-bit state */ static void KeccakF1600Step(SHA3Context *p){ … } /* ** Initialize a new hash. iSize determines the size of the hash ** in bits and should be one of 224, 256, 384, or 512. Or iSize ** can be zero to use the default hash size of 256 bits. */ static void SHA3Init(SHA3Context *p, int iSize){ … } /* ** Make consecutive calls to the SHA3Update function to add new content ** to the hash */ static void SHA3Update( SHA3Context *p, const unsigned char *aData, unsigned int nData ){ … } /* ** After all content has been added, invoke SHA3Final() to compute ** the final hash. The function returns a pointer to the binary ** hash value. */ static unsigned char *SHA3Final(SHA3Context *p){ … } /* End of the hashing logic *****************************************************************************/ /* ** Implementation of the sha3(X,SIZE) function. ** ** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default ** size is 256. If X is a BLOB, it is hashed as is. ** For all other non-NULL types of input, X is converted into a UTF-8 string ** and the string is hashed without the trailing 0x00 terminator. The hash ** of a NULL value is NULL. */ static void sha3Func( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* Compute a string using sqlite3_vsnprintf() with a maximum length ** of 50 bytes and add it to the hash. */ static void sha3_step_vformat( SHA3Context *p, /* Add content to this context */ const char *zFormat, ... ){ … } /* ** Implementation of the sha3_query(SQL,SIZE) function. ** ** This function compiles and runs the SQL statement(s) given in the ** argument. The results are hashed using a SIZE-bit SHA3. The default ** size is 256. ** ** The format of the byte stream that is hashed is summarized as follows: ** ** S<n>:<sql> ** R ** N ** I<int> ** F<ieee-float> ** B<size>:<bytes> ** T<size>:<text> ** ** <sql> is the original SQL text for each statement run and <n> is ** the size of that text. The SQL text is UTF-8. A single R character ** occurs before the start of each row. N means a NULL value. ** I mean an 8-byte little-endian integer <int>. F is a floating point ** number with an 8-byte little-endian IEEE floating point value <ieee-float>. ** B means blobs of <size> bytes. T means text rendered as <size> ** bytes of UTF-8. The <n> and <size> values are expressed as an ASCII ** text integers. ** ** For each SQL statement in the X input, there is one S segment. Each ** S segment is followed by zero or more R segments, one for each row in the ** result set. After each R, there are one or more N, I, F, B, or T segments, ** one for each column in the result set. Segments are concatentated directly ** with no delimiters of any kind. */ static void sha3QueryFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #ifdef _WIN32 #endif int sqlite3_shathree_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/shathree.c ********************/ /************************* Begin ../ext/misc/uint.c ******************/ /* ** 2020-04-14 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This SQLite extension implements the UINT collating sequence. ** ** UINT works like BINARY for text, except that embedded strings ** of digits compare in numeric order. ** ** * Leading zeros are handled properly, in the sense that ** they do not mess of the maginitude comparison of embedded ** strings of digits. "x00123y" is equal to "x123y". ** ** * Only unsigned integers are recognized. Plus and minus ** signs are ignored. Decimal points and exponential notation ** are ignored. ** ** * Embedded integers can be of arbitrary length. Comparison ** is *not* limited integers that can be expressed as a ** 64-bit machine integer. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> #include <ctype.h> /* ** Compare text in lexicographic order, except strings of digits ** compare in numeric order. */ static int uintCollFunc( void *notUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ … } #ifdef _WIN32 #endif int sqlite3_uint_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/uint.c ********************/ /************************* Begin ../ext/misc/decimal.c ******************/ /* ** 2020-06-22 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** Routines to implement arbitrary-precision decimal math. ** ** The focus here is on simplicity and correctness, not performance. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> #include <ctype.h> #include <stdlib.h> /* Mark a function parameter as unused, to suppress nuisance compiler ** warnings. */ #ifndef UNUSED_PARAMETER #define UNUSED_PARAMETER … #endif /* A decimal object */ Decimal; struct Decimal { … }; /* ** Release memory held by a Decimal, but do not free the object itself. */ static void decimal_clear(Decimal *p){ … } /* ** Destroy a Decimal object */ static void decimal_free(Decimal *p){ … } /* ** Allocate a new Decimal object initialized to the text in zIn[]. ** Return NULL if any kind of error occurs. */ static Decimal *decimalNewFromText(const char *zIn, int n){ … } /* Forward reference */ static Decimal *decimalFromDouble(double); /* ** Allocate a new Decimal object from an sqlite3_value. Return a pointer ** to the new object, or NULL if there is an error. If the pCtx argument ** is not NULL, then errors are reported on it as well. ** ** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted ** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length ** 8 bytes, the resulting double value is expanded into its decimal equivalent. ** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, ** then NULL is returned. */ static Decimal *decimal_new( sqlite3_context *pCtx, /* Report error here, if not null */ sqlite3_value *pIn, /* Construct the decimal object from this */ int bTextOnly /* Always interpret pIn as text if true */ ){ … } /* ** Make the given Decimal the result. */ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ … } /* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ … } /* ** Compare to Decimal objects. Return negative, 0, or positive if the ** first object is less than, equal to, or greater than the second. ** ** Preconditions for this routine: ** ** pA!=0 ** pA->isNull==0 ** pB!=0 ** pB->isNull==0 */ static int decimal_cmp(const Decimal *pA, const Decimal *pB){ … } /* ** SQL Function: decimal_cmp(X, Y) ** ** Return negative, zero, or positive if X is less then, equal to, or ** greater than Y. */ static void decimalCmpFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** Expand the Decimal so that it has a least nDigit digits and nFrac ** digits to the right of the decimal point. */ static void decimal_expand(Decimal *p, int nDigit, int nFrac){ … } /* ** Add the value pB into pA. A := A + B. ** ** Both pA and pB might become denormalized by this routine. */ static void decimal_add(Decimal *pA, Decimal *pB){ … } /* ** Multiply A by B. A := A * B ** ** All significant digits after the decimal point are retained. ** Trailing zeros after the decimal point are omitted as long as ** the number of digits after the decimal point is no less than ** either the number of digits in either input. */ static void decimalMul(Decimal *pA, Decimal *pB){ … } /* ** Create a new Decimal object that contains an integer power of 2. */ static Decimal *decimalPow2(int N){ … } /* ** Use an IEEE754 binary64 ("double") to generate a new Decimal object. */ static Decimal *decimalFromDouble(double r){ … } /* ** SQL Function: decimal(X) ** OR: decimal_exp(X) ** ** Convert input X into decimal and then back into text. ** ** If X is originally a float, then a full decimal expansion of that floating ** point value is done. Or if X is an 8-byte blob, it is interpreted ** as a float and similarly expanded. ** ** The decimal_exp(X) function returns the result in exponential notation. ** decimal(X) returns a complete decimal, without the e+NNN at the end. */ static void decimalFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** Compare text in decimal order. */ static int decimalCollFunc( void *notUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ … } /* ** SQL Function: decimal_add(X, Y) ** decimal_sub(X, Y) ** ** Return the sum or difference of X and Y. */ static void decimalAddFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } static void decimalSubFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* Aggregate funcion: decimal_sum(X) ** ** Works like sum() except that it uses decimal arithmetic for unlimited ** precision. */ static void decimalSumStep( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } static void decimalSumInverse( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } static void decimalSumValue(sqlite3_context *context){ … } static void decimalSumFinalize(sqlite3_context *context){ … } /* ** SQL Function: decimal_mul(X, Y) ** ** Return the product of X and Y. */ static void decimalMulFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** SQL Function: decimal_pow2(N) ** ** Return the N-th power of 2. N must be an integer. */ static void decimalPow2Func( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #ifdef _WIN32 #endif int sqlite3_decimal_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/decimal.c ********************/ #undef sqlite3_base_init #define sqlite3_base_init … /************************* Begin ../ext/misc/base64.c ******************/ /* ** 2022-11-18 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a ** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** ** This is an independent implementation of conversions specified in ** RFC 4648, done on the above date by the author (Larry Brasfield) ** who thereby has the right to put this into the public domain. ** ** The conversions meet RFC 4648 requirements, provided that this ** C source specifies that line-feeds are included in the encoded ** data to limit visible line lengths to 72 characters and to ** terminate any encoded blob having non-zero length. ** ** Length limitations are not imposed except that the runtime ** SQLite string or blob length limits are respected. Otherwise, ** any length binary sequence can be represented and recovered. ** Generated base64 sequences, with their line-feeds included, ** can be concatenated; the result converted back to binary will ** be the concatenation of the represented binary sequences. ** ** This SQLite3 extension creates a function, base64(x), which ** either: converts text x containing base64 to a returned blob; ** or converts a blob x to returned text containing base64. An ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the ** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. ** Because it is table-driven, it could be enhanced to handle that, ** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR=<your favorite SQLite checkout directory> ** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c ** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c ** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c ** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll */ #include <assert.h> /* #include "sqlite3ext.h" */ #ifndef deliberate_fall_through /* Quiet some compilers about some of our intentional code. */ # if GCC_VERSION>=7000000 #define deliberate_fall_through … # else #define deliberate_fall_through # endif #endif SQLITE_EXTENSION_INIT1; #define PC … #define WS … #define ND … #define PAD_CHAR … #ifndef U8_TYPEDEF /* typedef unsigned char u8; */ #define U8_TYPEDEF #endif /* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ static const u8 b64DigitValues[128] = …; static const char b64Numerals[64+1] = …; #define BX_DV_PROTO(c) … #define IS_BX_DIGIT(bdp) … #define IS_BX_WS(bdp) … #define IS_BX_PAD(bdp) … #define BX_NUMERAL(dv) … /* Width of base64 lines. Should be an integer multiple of 4. */ #define B64_DARK_MAX … /* Encode a byte buffer into base64 text with linefeeds appended to limit ** encoded group lengths to B64_DARK_MAX or to terminate the last group. */ static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ … } /* Skip over text which is not base64 numeral(s). */ static char * skipNonB64( char *s, int nc ){ … } /* Decode base64 text into a byte buffer. */ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ … } /* This function does the work for the SQLite base64(x) UDF. */ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ … } /* ** Establish linkage to running SQLite library. */ #ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 #endif int sqlite3_base_init #else static int sqlite3_base64_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ … } /* ** Define some macros to allow this extension to be built into the shell ** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This ** allows shell.c, as distributed, to have this extension built in. */ #define BASE64_INIT(db) … #define BASE64_EXPOSE(db, pzErr) … /************************* End ../ext/misc/base64.c ********************/ #undef sqlite3_base_init #define sqlite3_base_init … #define OMIT_BASE85_CHECKER /************************* Begin ../ext/misc/base85.c ******************/ /* ** 2022-11-16 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This is a utility for converting binary to base85 or vice-versa. ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a ** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** ** The encoding used resembles Ascii85, but was devised by the author ** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 ** variant sources existed, in the 1984 timeframe on a VAX mainframe. ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** ** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** ** Groups of 4 bytes, interpreted as big-endian 32-bit values, ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields ** the simplest transition to byte groups smaller than 4 bytes. ** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. ** No pad characters are used; Encoded base85 numeral sequence ** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite ** size, newline is used to separate it into sub-sequences of no ** more than 80 digits so that fgets() can be used to read it. ** ** Length limitations are not imposed except that the runtime ** SQLite string or blob length limits are respected. Otherwise, ** any length binary sequence can be represented and recovered. ** Base85 sequences can be concatenated by separating them with ** a non-base85 character; the conversion to binary will then ** be the concatenation of the represented binary sequences. ** The standalone program either converts base85 on stdin to create ** a binary file or converts a binary file to base85 on stdout. ** Read or make it blurt its help for invocation details. ** ** The SQLite3 extension creates a function, base85(x), which will ** either convert text base85 to a blob or a blob to text base85 ** and return the result (or throw an error for other types.) ** Unless built with OMIT_BASE85_CHECKER defined, it also creates a ** function, is_base85(t), which returns 1 iff the text t contains ** nothing other than base85 numerals and whitespace, or 0 otherwise. ** ** To build the extension: ** Set shell variable SQDIR=<your favorite SQLite checkout directory> ** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. ** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c ** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c ** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c ** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll ** ** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. ** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 ** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c ** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c */ #include <stdio.h> #include <memory.h> #include <string.h> #include <assert.h> #ifndef OMIT_BASE85_CHECKER # include <ctype.h> #endif #ifndef BASE85_STANDALONE /* # include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1; #else # ifdef _WIN32 # include <io.h> # include <fcntl.h> # else #define setmode … # endif static char *zHelp = "Usage: base85 <dirFlag> <binFile>\n" " <dirFlag> is either -r to read or -w to write <binFile>,\n" " content to be converted to/from base85 on stdout/stdin.\n" " <binFile> names a binary file to be rendered or created.\n" " Or, the name '-' refers to the stdin or stdout stream.\n" ; static void sayHelp(){ printf("%s", zHelp); } #endif #ifndef U8_TYPEDEF /* typedef unsigned char u8; */ #define U8_TYPEDEF #endif /* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) … /* Provide digitValue to b85Numeral offset as a function of above class. */ static u8 b85_cOffset[] = …; #define B85_DNOS( c ) … /* Say whether c is a base85 numeral. */ #define IS_B85( c ) … #if 0 /* Not used, */ static u8 base85DigitValue( char c ){ u8 dv = (u8)(c - '#'); if( dv>87 ) return 0xff; return (dv > 3)? dv-3 : dv; } #endif /* Width of base64 lines. Should be an integer multiple of 5. */ #define B85_DARK_MAX … static char * skipNonB85( char *s, int nc ){ … } /* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. * Do not use the macro form with argument expression having a side-effect.*/ #if 0 static char base85Numeral( u8 b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } #else #define base85Numeral( dn ) … #endif static char *putcs(char *pc, char *s){ … } /* Encode a byte buffer into base85 text. If pSep!=0, it's a C string ** to be appended to encoded groups to limit their length to B85_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ … } /* Decode base85 text into a byte buffer. */ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ … } #ifndef OMIT_BASE85_CHECKER /* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ if( !IS_B85(c) && !isspace(c) ) return 0; } return 1; } #endif #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER /* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ case SQLITE_TEXT: { int rv = allBase85( (char *)sqlite3_value_text(av[0]), sqlite3_value_bytes(av[0]) ); sqlite3_result_int(context, rv); } break; case SQLITE_NULL: sqlite3_result_null(context); break; default: sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); return; } } # endif /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ … } /* ** Establish linkage to running SQLite library. */ #ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 #endif int sqlite3_base_init #else static int sqlite3_base85_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ … } /* ** Define some macros to allow this extension to be built into the shell ** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This ** allows shell.c, as distributed, to have this extension built in. */ #define BASE85_INIT(db) … #define BASE85_EXPOSE(db, pzErr) … #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; u8 bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER int b85Clean = 1; # endif char rw; FILE *fb = 0, *foc = 0; char fmode[3] = "xb"; if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ sayHelp(); return 0; } fmode[0] = rw; if( av[2][0]=='-' && av[2][1]==0 ){ switch( rw ){ case 'r': fb = stdin; setmode(fileno(stdin), O_BINARY); break; case 'w': fb = stdout; setmode(fileno(stdout), O_BINARY); break; } }else{ fb = fopen(av[2], fmode); foc = fb; } if( !fb ){ fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); rc = 1; }else{ switch( rw ){ case 'r': while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ toBase85( bBuf, (int)nio, cBuf, 0 ); fprintf(stdout, "%s\n", cBuf); } break; case 'w': while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ int nc = strlen(cBuf); size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; # ifndef OMIT_BASE85_CHECKER b85Clean &= allBase85( cBuf, nc ); # endif } break; default: sayHelp(); rc = 1; } if( foc ) fclose(foc); } # ifndef OMIT_BASE85_CHECKER if( !b85Clean ){ fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); } # endif return rc; } #endif /************************* End ../ext/misc/base85.c ********************/ /************************* Begin ../ext/misc/ieee754.c ******************/ /* ** 2013-04-17 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This SQLite extension implements functions for the exact display ** and input of IEEE754 Binary64 floating-point numbers. ** ** ieee754(X) ** ieee754(Y,Z) ** ** In the first form, the value X should be a floating-point number. ** The function will return a string of the form 'ieee754(Y,Z)' where ** Y and Z are integers such that X==Y*pow(2,Z). ** ** In the second form, Y and Z are integers which are the mantissa and ** base-2 exponent of a new floating point number. The function returns ** a floating-point value equal to Y*pow(2,Z). ** ** Examples: ** ** ieee754(2.0) -> 'ieee754(2,0)' ** ieee754(45.25) -> 'ieee754(181,-2)' ** ieee754(2, 0) -> 2.0 ** ieee754(181, -2) -> 45.25 ** ** Two additional functions break apart the one-argument ieee754() ** result into separate integer values: ** ** ieee754_mantissa(45.25) -> 181 ** ieee754_exponent(45.25) -> -2 ** ** These functions convert binary64 numbers into blobs and back again. ** ** ieee754_from_blob(x'3ff0000000000000') -> 1.0 ** ieee754_to_blob(1.0) -> x'3ff0000000000000' ** ** In all single-argument functions, if the argument is an 8-byte blob ** then that blob is interpreted as a big-endian binary64 value. ** ** ** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES ** ----------------------------------------------- ** ** This extension in combination with the separate 'decimal' extension ** can be used to compute the exact decimal representation of binary64 ** values. To begin, first compute a table of exponent values: ** ** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); ** WITH RECURSIVE c(x,v) AS ( ** VALUES(0,'1') ** UNION ALL ** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 ** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; ** WITH RECURSIVE c(x,v) AS ( ** VALUES(-1,'0.5') ** UNION ALL ** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 ** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; ** ** Then, to compute the exact decimal representation of a floating ** point value (the value 47.49 is used in the example) do: ** ** WITH c(n) AS (VALUES(47.49)) ** ---------------^^^^^---- Replace with whatever you want ** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) ** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); ** ** Here is a query to show various boundry values for the binary64 ** number format: ** ** WITH c(name,bin) AS (VALUES ** ('minimum positive value', x'0000000000000001'), ** ('maximum subnormal value', x'000fffffffffffff'), ** ('mininum positive nornal value', x'0010000000000000'), ** ('maximum value', x'7fefffffffffffff')) ** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) ** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); ** */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> /* Mark a function parameter as unused, to suppress nuisance compiler ** warnings. */ #ifndef UNUSED_PARAMETER #define UNUSED_PARAMETER … #endif /* ** Implementation of the ieee754() function */ static void ieee754func( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** Functions to convert between blobs and floats. */ static void ieee754func_from_blob( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } static void ieee754func_to_blob( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** SQL Function: ieee754_inc(r,N) ** ** Move the floating point value r by N quantums and return the new ** values. ** ** Behind the scenes: this routine merely casts r into a 64-bit unsigned ** integer, adds N, then casts the value back into float. ** ** Example: To find the smallest positive number: ** ** SELECT ieee754_inc(0.0,+1); */ static void ieee754inc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #ifdef _WIN32 #endif int sqlite3_ieee_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/ieee754.c ********************/ /************************* Begin ../ext/misc/series.c ******************/ /* ** 2015-08-18, 2023-04-28 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file demonstrates how to create a table-valued-function using ** a virtual table. This demo implements the generate_series() function ** which gives the same results as the eponymous function in PostgreSQL, ** within the limitation that its arguments are signed 64-bit integers. ** ** Considering its equivalents to generate_series(start,stop,step): A ** value V[n] sequence is produced for integer n ascending from 0 where ** ( V[n] == start + n * step && sgn(V[n] - stop) * sgn(step) >= 0 ) ** for each produced value (independent of production time ordering.) ** ** All parameters must be either integer or convertable to integer. ** The start parameter is required. ** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) ** The step parameter defaults to 1 and 0 is treated as 1. ** ** Examples: ** ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps ** of 5. ** ** SELECT * FROM generate_series(0,100); ** ** Integers from 0 through 100 with a step size of 1. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** ** Integers 20 through 29. ** ** SELECT * FROM generate_series(0,-100,-5); ** ** Integers 0 -5 -10 ... -100. ** ** SELECT * FROM generate_series(0,-1); ** ** Empty sequence. ** ** HOW IT WORKS ** ** The generate_series "function" is really a virtual table with the ** following schema: ** ** CREATE TABLE generate_series( ** value, ** start HIDDEN, ** stop HIDDEN, ** step HIDDEN ** ); ** ** The virtual table also has a rowid, logically equivalent to n+1 where ** "n" is the ascending integer in the aforesaid production definition. ** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other ** words, the following pairs of queries are equivalent to each other: ** ** SELECT * FROM generate_series(0,100,5); ** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; ** ** SELECT * FROM generate_series(0,100); ** SELECT * FROM generate_series WHERE start=0 AND stop=100; ** ** SELECT * FROM generate_series(20) LIMIT 10; ** SELECT * FROM generate_series WHERE start=20 LIMIT 10; ** ** The generate_series virtual table implementation leaves the xCreate method ** set to NULL. This means that it is not possible to do a CREATE VIRTUAL ** TABLE command with "generate_series" as the USING argument. Instead, there ** is a single generate_series virtual table that is always available without ** having to be created first. ** ** The xBestIndex method looks for equality constraints against the hidden ** start, stop, and step columns, and if present, it uses those constraints ** to bound the sequence of generated values. If the equality constraints ** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. ** xBestIndex returns a small cost when both start and stop are available, ** and a very large cost if either start or stop are unavailable. This ** encourages the query planner to order joins such that the bounds of the ** series are well-defined. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> #include <limits.h> #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Return that member of a generate_series(...) sequence whose 0-based ** index is ix. The 0th member is given by smBase. The sequence members ** progress per ix increment by smStep. */ static sqlite3_int64 genSeqMember( sqlite3_int64 smBase, sqlite3_int64 smStep, sqlite3_uint64 ix ){ … } /* typedef unsigned char u8; */ SequenceSpec; /* ** Prepare a SequenceSpec for use in generating an integer series ** given initialized iBase, iTerm and iStep values. Sequence is ** initialized per given isReversing. Other members are computed. */ static void setupSequence( SequenceSpec *pss ){ … } /* ** Progress sequence generator to yield next value, if any. ** Leave its state to either yield next value or be at EOF. ** Return whether there is a next value, or 0 at EOF. */ static int progressSequence( SequenceSpec *pss ){ … } /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans ** over rows of the result */ series_cursor; struct series_cursor { … }; /* ** The seriesConnect() method is invoked to create a new ** series_vtab that describes the generate_series virtual table. ** ** Think of this routine as the constructor for series_vtab objects. ** ** All this routine needs to do is: ** ** (1) Allocate the series_vtab object and initialize all fields. ** ** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the ** result set of queries against generate_series will look like. */ static int seriesConnect( sqlite3 *db, void *pUnused, int argcUnused, const char *const*argvUnused, sqlite3_vtab **ppVtab, char **pzErrUnused ){ … } /* ** This method is the destructor for series_cursor objects. */ static int seriesDisconnect(sqlite3_vtab *pVtab){ … } /* ** Constructor for a new series_cursor object. */ static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ … } /* ** Destructor for a series_cursor. */ static int seriesClose(sqlite3_vtab_cursor *cur){ … } /* ** Advance a series_cursor to its next row of output. */ static int seriesNext(sqlite3_vtab_cursor *cur){ … } /* ** Return values of columns for the row at which the series_cursor ** is currently pointing. */ static int seriesColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ … } #ifndef LARGEST_UINT64 #define LARGEST_UINT64 … #endif /* ** Return the rowid for the current row, logically equivalent to n+1 where ** "n" is the ascending integer in the aforesaid production definition. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ … } /* ** Return TRUE if the cursor has been moved off of the last ** row of output. */ static int seriesEof(sqlite3_vtab_cursor *cur){ … } /* True to cause run-time checking of the start=, stop=, and/or step= ** parameters. The only reason to do this is for testing the ** constraint checking logic for virtual tables in the SQLite core. */ #ifndef SQLITE_SERIES_CONSTRAINT_VERIFY #define SQLITE_SERIES_CONSTRAINT_VERIFY … #endif /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least ** once prior to any call to seriesColumn() or seriesRowid() or ** seriesEof(). ** ** The query plan selected by seriesBestIndex is passed in the idxNum ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** ** 0x01: start=VALUE ** 0x02: stop=VALUE ** 0x04: step=VALUE ** 0x08: descending order ** 0x10: ascending order ** 0x20: LIMIT VALUE ** 0x40: OFFSET VALUE ** ** This routine should initialize the cursor and position it so that it ** is pointing at the first row, or pointing off the end of the table ** (so that seriesEof() will return true) if the table is empty. */ static int seriesFilter( sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStrUnused, int argc, sqlite3_value **argv ){ … } /* ** SQLite will invoke this method one or more times while planning a query ** that uses the generate_series virtual table. This routine needs to create ** a query plan for each invocation and compute an estimated cost for that ** plan. ** ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** ** The query plan is represented by bits in idxNum: ** ** 0x01 start = $value -- constraint exists ** 0x02 stop = $value -- constraint exists ** 0x04 step = $value -- constraint exists ** 0x08 output is in descending order ** 0x10 output is in ascending order ** 0x20 LIMIT $value -- constraint exists ** 0x40 OFFSET $value -- constraint exists */ static int seriesBestIndex( sqlite3_vtab *pVTab, sqlite3_index_info *pIdxInfo ){ … } /* ** This following structure defines all the methods for the ** generate_series virtual table. */ static sqlite3_module seriesModule = …; #endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifdef _WIN32 #endif int sqlite3_series_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/series.c ********************/ /************************* Begin ../ext/misc/regexp.c ******************/ /* ** 2012-11-13 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** The code in this file implements a compact but reasonably ** efficient regular-expression matcher for posix extended regular ** expressions against UTF8 text. ** ** This file is an SQLite extension. It registers a single function ** named "regexp(A,B)" where A is the regular expression and B is the ** string to be matched. By registering this function, SQLite will also ** then implement the "B regexp A" operator. Note that with the function ** the regular expression comes first, but with the operator it comes ** second. ** ** The following regular expression syntax is supported: ** ** X* zero or more occurrences of X ** X+ one or more occurrences of X ** X? zero or one occurrences of X ** X{p,q} between p and q occurrences of X ** (X) match X ** X|Y X or Y ** ^X X occurring at the beginning of the string ** X$ X occurring at the end of the string ** . Match any single character ** \c Character c where c is one of \{}()[]|*+?. ** \c C-language escapes for c in afnrtv. ex: \t or \n ** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX ** \xXX Where XX is exactly 2 hex digits, unicode value XX ** [abc] Any single character from the set abc ** [^abc] Any single character not in the set abc ** [a-z] Any single character in the range a-z ** [^a-z] Any single character not in the range a-z ** \b Word boundary ** \w Word character. [A-Za-z0-9_] ** \W Non-word character ** \d Digit ** \D Non-digit ** \s Whitespace character ** \S Non-whitespace character ** ** A nondeterministic finite automaton (NFA) is used for matching, so the ** performance is bounded by O(N*M) where N is the size of the regular ** expression and M is the size of the input string. The matcher never ** exhibits exponential behavior. Note that the X{p,q} operator expands ** to p copies of X following by q-p copies of X? and that the size of the ** regular expression in the O(N*M) performance bound is computed after ** this expansion. */ #include <string.h> #include <stdlib.h> /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 /* ** The following #defines change the names of some functions implemented in ** this file to prevent name collisions with C-library functions of the ** same name. */ #define re_match … #define re_compile … #define re_free … /* The end-of-input character */ #define RE_EOF … #define RE_START … /* The NFA is implemented as sequence of opcodes taken from the following ** set. Each opcode has a single integer argument. */ #define RE_OP_MATCH … #define RE_OP_ANY … #define RE_OP_ANYSTAR … #define RE_OP_FORK … #define RE_OP_GOTO … #define RE_OP_ACCEPT … #define RE_OP_CC_INC … #define RE_OP_CC_EXC … #define RE_OP_CC_VALUE … #define RE_OP_CC_RANGE … #define RE_OP_WORD … #define RE_OP_NOTWORD … #define RE_OP_DIGIT … #define RE_OP_NOTDIGIT … #define RE_OP_SPACE … #define RE_OP_NOTSPACE … #define RE_OP_BOUNDARY … #define RE_OP_ATSTART … #if defined(SQLITE_DEBUG) /* Opcode names used for symbolic debugging */ static const char *ReOpName[] = { "EOF", "MATCH", "ANY", "ANYSTAR", "FORK", "GOTO", "ACCEPT", "CC_INC", "CC_EXC", "CC_VALUE", "CC_RANGE", "WORD", "NOTWORD", "DIGIT", "NOTDIGIT", "SPACE", "NOTSPACE", "BOUNDARY", "ATSTART", }; #endif /* SQLITE_DEBUG */ /* Each opcode is a "state" in the NFA */ ReStateNumber; /* Because this is an NFA and not a DFA, multiple states can be active at ** once. An instance of the following object records all active states in ** the NFA. The implementation is optimized for the common case where the ** number of actives states is small. */ ReStateSet; /* An input string read one character at a time. */ ReInput; struct ReInput { … }; /* A compiled NFA (or an NFA that is in the process of being compiled) is ** an instance of the following object. */ ReCompiled; struct ReCompiled { … }; /* Add a state to the given state set if it is not already there */ static void re_add_state(ReStateSet *pSet, int newState){ … } /* Extract the next unicode character from *pzIn and return it. Advance ** *pzIn to the first byte past the end of the character returned. To ** be clear: this routine converts utf8 to unicode. This routine is ** optimized for the common case where the next character is a single byte. */ static unsigned re_next_char(ReInput *p){ … } static unsigned re_next_char_nocase(ReInput *p){ … } /* Return true if c is a perl "word" character: [A-Za-z0-9_] */ static int re_word_char(int c){ … } /* Return true if c is a "digit" character: [0-9] */ static int re_digit_char(int c){ … } /* Return true if c is a perl "space" character: [ \t\r\n\v\f] */ static int re_space_char(int c){ … } /* Run a compiled regular expression on the zero-terminated input ** string zIn[]. Return true on a match and false if there is no match. */ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ … } /* Resize the opcode and argument arrays for an RE under construction. */ static int re_resize(ReCompiled *p, int N){ … } /* Insert a new opcode and argument into an RE under construction. The ** insertion point is just prior to existing opcode iBefore. */ static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ … } /* Append a new opcode and argument to the end of the RE under construction. */ static int re_append(ReCompiled *p, int op, int arg){ … } /* Make a copy of N opcodes starting at iStart onto the end of the RE ** under construction. */ static void re_copy(ReCompiled *p, int iStart, int N){ … } /* Return true if c is a hexadecimal digit character: [0-9a-fA-F] ** If c is a hex digit, also set *pV = (*pV)*16 + valueof(c). If ** c is not a hex digit *pV is unchanged. */ static int re_hex(int c, int *pV){ … } /* A backslash character has been seen, read the next character and ** return its interpretation. */ static unsigned re_esc_char(ReCompiled *p){ … } /* Forward declaration */ static const char *re_subcompile_string(ReCompiled*); /* Peek at the next byte of input */ static unsigned char rePeek(ReCompiled *p){ … } /* Compile RE text into a sequence of opcodes. Continue up to the ** first unmatched ")" character, then return. If an error is found, ** return a pointer to the error message string. */ static const char *re_subcompile_re(ReCompiled *p){ … } /* Compile an element of regular expression text (anything that can be ** an operand to the "|" operator). Return NULL on success or a pointer ** to the error message if there is a problem. */ static const char *re_subcompile_string(ReCompiled *p){ … } /* Free and reclaim all the memory used by a previously compiled ** regular expression. Applications should invoke this routine once ** for every call to re_compile() to avoid memory leaks. */ static void re_free(ReCompiled *pRe){ … } /* ** Compile a textual regular expression in zIn[] into a compiled regular ** expression suitable for us by re_match() and return a pointer to the ** compiled regular expression in *ppRe. Return NULL on success or an ** error message if something goes wrong. */ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ … } /* ** Implementation of the regexp() SQL function. This function implements ** the build-in REGEXP operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: ** ** A REGEXP B ** ** is implemented as regexp(B,A). */ static void re_sql_func( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #if defined(SQLITE_DEBUG) /* ** This function is used for testing and debugging only. It is only available ** if the SQLITE_DEBUG compile-time option is used. ** ** Compile a regular expression and then convert the compiled expression into ** text and return that text. */ static void re_bytecode_func( sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zPattern; const char *zErr; ReCompiled *pRe; sqlite3_str *pStr; int i; int n; char *z; (void)argc; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); return; } if( pRe==0 ){ sqlite3_result_error_nomem(context); return; } pStr = sqlite3_str_new(0); if( pStr==0 ) goto re_bytecode_func_err; if( pRe->nInit>0 ){ sqlite3_str_appendf(pStr, "INIT "); for(i=0; i<pRe->nInit; i++){ sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]); } sqlite3_str_appendf(pStr, "\n"); } for(i=0; (unsigned)i<pRe->nState; i++){ sqlite3_str_appendf(pStr, "%-8s %4d\n", ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]); } n = sqlite3_str_length(pStr); z = sqlite3_str_finish(pStr); if( n==0 ){ sqlite3_free(z); }else{ sqlite3_result_text(context, z, n-1, sqlite3_free); } re_bytecode_func_err: re_free(pRe); } #endif /* SQLITE_DEBUG */ /* ** Invoke this routine to register the regexp() function with the ** SQLite database connection. */ #ifdef _WIN32 #endif int sqlite3_regexp_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/regexp.c ********************/ #ifndef SQLITE_SHELL_FIDDLE /************************* Begin ../ext/misc/fileio.c ******************/ /* ** 2014-06-13 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This SQLite extension implements SQL functions readfile() and ** writefile(), and eponymous virtual type "fsdir". ** ** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): ** ** If neither of the optional arguments is present, then this UDF ** function writes blob DATA to file FILE. If successful, the number ** of bytes written is returned. If an error occurs, NULL is returned. ** ** If the first option argument - MODE - is present, then it must ** be passed an integer value that corresponds to a POSIX mode ** value (file type + permissions, as returned in the stat.st_mode ** field by the stat() system call). Three types of files may ** be written/created: ** ** regular files: (mode & 0170000)==0100000 ** symbolic links: (mode & 0170000)==0120000 ** directories: (mode & 0170000)==0040000 ** ** For a directory, the DATA is ignored. For a symbolic link, it is ** interpreted as text and used as the target of the link. For a ** regular file, it is interpreted as a blob and written into the ** named file. Regardless of the type of file, its permissions are ** set to (mode & 0777) before returning. ** ** If the optional MTIME argument is present, then it is interpreted ** as an integer - the number of seconds since the unix epoch. The ** modification-time of the target file is set to this value before ** returning. ** ** If three or more arguments are passed to this function and an ** error is encountered, an exception is raised. ** ** READFILE(FILE): ** ** Read and return the contents of file FILE (type blob) from disk. ** ** FSDIR: ** ** Used as follows: ** ** SELECT * FROM fsdir($path [, $dir]); ** ** Parameter $path is an absolute or relative pathname. If the file that it ** refers to does not exist, it is an error. If the path refers to a regular ** file or symbolic link, it returns a single row. Or, if the path refers ** to a directory, it returns one row for the directory, and one row for each ** file within the hierarchy rooted at $path. ** ** Each row has the following columns: ** ** name: Path to file or directory (text value). ** mode: Value of stat.st_mode for directory entry (an integer). ** mtime: Value of stat.st_mtime for directory entry (an integer). ** data: For a regular file, a blob containing the file data. For a ** symlink, a text value containing the text of the link. For a ** directory, NULL. ** ** If a non-NULL value is specified for the optional $dir parameter and ** $path is a relative path, then $path is interpreted relative to $dir. ** And the paths returned in the "name" column of the table are also ** relative to directory $dir. ** ** Notes on building this extension for Windows: ** Unless linked statically with the SQLite library, a preprocessor ** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone ** DLL form of this extension for WIN32. See its use below for details. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <stdio.h> #include <string.h> #include <assert.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> #if !defined(_WIN32) && !defined(WIN32) # include <unistd.h> # include <dirent.h> # include <utime.h> # include <sys/time.h> #else # include "windows.h" # include <io.h> # include <direct.h> /* # include "test_windirent.h" */ #define dirent … # ifndef chmod #define chmod … # endif # ifndef stat #define stat … # endif #define mkdir … #define lstat … #endif #include <time.h> #include <errno.h> /* ** Structure of the fsdir() table-valued function */ /* 0 1 2 3 4 5 */ #define FSDIR_SCHEMA … #define FSDIR_COLUMN_NAME … #define FSDIR_COLUMN_MODE … #define FSDIR_COLUMN_MTIME … #define FSDIR_COLUMN_DATA … #define FSDIR_COLUMN_PATH … #define FSDIR_COLUMN_DIR … /* ** Set the result stored by context ctx to a blob containing the ** contents of file zName. Or, leave the result unchanged (NULL) ** if the file does not exist or is unreadable. ** ** If the file exceeds the SQLite blob size limit, through an ** SQLITE_TOOBIG error. ** ** Throw an SQLITE_IOERR if there are difficulties pulling the file ** off of disk. */ static void readFileContents(sqlite3_context *ctx, const char *zName){ … } /* ** Implementation of the "readfile(X)" SQL function. The entire content ** of the file named X is read and returned as a BLOB. NULL is returned ** if the file does not exist or is unreadable. */ static void readfileFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** Set the error message contained in context ctx to the results of ** vprintf(zFmt, ...). */ static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ … } #if defined(_WIN32) /* ** This function is designed to convert a Win32 FILETIME structure into the ** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). */ static sqlite3_uint64 fileTimeToUnixTime( LPFILETIME pFileTime ){ SYSTEMTIME epochSystemTime; ULARGE_INTEGER epochIntervals; FILETIME epochFileTime; ULARGE_INTEGER fileIntervals; memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); epochSystemTime.wYear = 1970; epochSystemTime.wMonth = 1; epochSystemTime.wDay = 1; SystemTimeToFileTime(&epochSystemTime, &epochFileTime); epochIntervals.LowPart = epochFileTime.dwLowDateTime; epochIntervals.HighPart = epochFileTime.dwHighDateTime; fileIntervals.LowPart = pFileTime->dwLowDateTime; fileIntervals.HighPart = pFileTime->dwHighDateTime; return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } #if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) # /* To allow a standalone DLL, use this next replacement function: */ # undef sqlite3_win32_utf8_to_unicode #define sqlite3_win32_utf8_to_unicode … # LPWSTR utf8_to_utf16(const char *z){ int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) return rv; sqlite3_free(rv); return 0; } #endif /* ** This function attempts to normalize the time values found in the stat() ** buffer to UTC. This is necessary on Win32, where the runtime library ** appears to return these values as local times. */ static void statTimesToUtc( const char *zPath, struct stat *pStatBuf ){ HANDLE hFindFile; WIN32_FIND_DATAW fd; LPWSTR zUnicodeName; extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); if( zUnicodeName ){ memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); hFindFile = FindFirstFileW(zUnicodeName, &fd); if( hFindFile!=NULL ){ pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); FindClose(hFindFile); } sqlite3_free(zUnicodeName); } } #endif /* ** This function is used in place of stat(). On Windows, special handling ** is required in order for the included time to be returned as UTC. On all ** other systems, this function simply calls stat(). */ static int fileStat( const char *zPath, struct stat *pStatBuf ){ … } /* ** This function is used in place of lstat(). On Windows, special handling ** is required in order for the included time to be returned as UTC. On all ** other systems, this function simply calls lstat(). */ static int fileLinkStat( const char *zPath, struct stat *pStatBuf ){ … } /* ** Argument zFile is the name of a file that will be created and/or written ** by SQL function writefile(). This function ensures that the directory ** zFile will be written to exists, creating it if required. The permissions ** for any path components created by this function are set in accordance ** with the current umask. ** ** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, ** SQLITE_OK is returned if the directory is successfully created, or ** SQLITE_ERROR otherwise. */ static int makeDirectory( const char *zFile ){ … } /* ** This function does the work for the writefile() UDF. Refer to ** header comments at the top of this file for details. */ static int writeFile( sqlite3_context *pCtx, /* Context to return bytes written in */ const char *zFile, /* File to write */ sqlite3_value *pData, /* Data to write */ mode_t mode, /* MODE parameter passed to writefile() */ sqlite3_int64 mtime /* MTIME parameter (or -1 to not set time) */ ){ … } /* ** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. ** Refer to header comments at the top of this file for details. */ static void writefileFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } /* ** SQL function: lsmode(MODE) ** ** Given a numberic st_mode from stat(), convert it into a human-readable ** text string in the style of "ls -l". */ static void lsModeFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Cursor type for recursively iterating through a directory structure. */ fsdir_cursor; FsdirLevel; struct FsdirLevel { … }; struct fsdir_cursor { … }; fsdir_tab; struct fsdir_tab { … }; /* ** Construct a new fsdir virtual table object. */ static int fsdirConnect( sqlite3 *db, void *pAux, int argc, const char *const*argv, sqlite3_vtab **ppVtab, char **pzErr ){ … } /* ** This method is the destructor for fsdir vtab objects. */ static int fsdirDisconnect(sqlite3_vtab *pVtab){ … } /* ** Constructor for a new fsdir_cursor object. */ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ … } /* ** Reset a cursor back to the state it was in when first returned ** by fsdirOpen(). */ static void fsdirResetCursor(fsdir_cursor *pCur){ … } /* ** Destructor for an fsdir_cursor. */ static int fsdirClose(sqlite3_vtab_cursor *cur){ … } /* ** Set the error message for the virtual table associated with cursor ** pCur to the results of vprintf(zFmt, ...). */ static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ … } /* ** Advance an fsdir_cursor to its next row of output. */ static int fsdirNext(sqlite3_vtab_cursor *cur){ … } /* ** Return values of columns for the row at which the series_cursor ** is currently pointing. */ static int fsdirColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ … } /* ** Return the rowid for the current row. In this implementation, the ** first row returned is assigned rowid value 1, and each subsequent ** row a value 1 more than that of the previous. */ static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ … } /* ** Return TRUE if the cursor has been moved off of the last ** row of output. */ static int fsdirEof(sqlite3_vtab_cursor *cur){ … } /* ** xFilter callback. ** ** idxNum==1 PATH parameter only ** idxNum==2 Both PATH and DIR supplied */ static int fsdirFilter( sqlite3_vtab_cursor *cur, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ … } /* ** SQLite will invoke this method one or more times while planning a query ** that uses the generate_series virtual table. This routine needs to create ** a query plan for each invocation and compute an estimated cost for that ** plan. ** ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** ** The query plan is represented by values of idxNum: ** ** (1) The path value is supplied by argv[0] ** (2) Path is in argv[0] and dir is in argv[1] */ static int fsdirBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ … } /* ** Register the "fsdir" virtual table. */ static int fsdirRegister(sqlite3 *db){ … } #else /* SQLITE_OMIT_VIRTUALTABLE */ #define fsdirRegister … #endif #ifdef _WIN32 #endif int sqlite3_fileio_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } #if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) /* To allow a standalone DLL, make test_windirent.c use the same * redefined SQLite API calls as the above extension code does. * Just pull in this .c to accomplish this. As a beneficial side * effect, this extension becomes a single translation unit. */ # include "test_windirent.c" #endif /************************* End ../ext/misc/fileio.c ********************/ /************************* Begin ../ext/misc/completion.c ******************/ /* ** 2017-07-10 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file implements an eponymous virtual table that returns suggested ** completions for a partial SQL input. ** ** Suggested usage: ** ** SELECT DISTINCT candidate COLLATE nocase ** FROM completion($prefix,$wholeline) ** ORDER BY 1; ** ** The two query parameters are optional. $prefix is the text of the ** current word being typed and that is to be completed. $wholeline is ** the complete input line, used for context. ** ** The raw completion() table might return the same candidate multiple ** times, for example if the same column name is used to two or more ** tables. And the candidates are returned in an arbitrary order. Hence, ** the DISTINCT and ORDER BY are recommended. ** ** This virtual table operates at the speed of human typing, and so there ** is no attempt to make it fast. Even a slow implementation will be much ** faster than any human can type. ** */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <assert.h> #include <string.h> #include <ctype.h> #ifndef SQLITE_OMIT_VIRTUALTABLE /* completion_vtab is a subclass of sqlite3_vtab which will ** serve as the underlying representation of a completion virtual table */ completion_vtab; struct completion_vtab { … }; /* completion_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans ** over rows of the result */ completion_cursor; struct completion_cursor { … }; /* Values for ePhase: */ #define COMPLETION_FIRST_PHASE … #define COMPLETION_KEYWORDS … #define COMPLETION_PRAGMAS … #define COMPLETION_FUNCTIONS … #define COMPLETION_COLLATIONS … #define COMPLETION_INDEXES … #define COMPLETION_TRIGGERS … #define COMPLETION_DATABASES … #define COMPLETION_TABLES … #define COMPLETION_COLUMNS … #define COMPLETION_MODULES … #define COMPLETION_EOF … /* ** The completionConnect() method is invoked to create a new ** completion_vtab that describes the completion virtual table. ** ** Think of this routine as the constructor for completion_vtab objects. ** ** All this routine needs to do is: ** ** (1) Allocate the completion_vtab object and initialize all fields. ** ** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the ** result set of queries against completion will look like. */ static int completionConnect( sqlite3 *db, void *pAux, int argc, const char *const*argv, sqlite3_vtab **ppVtab, char **pzErr ){ … } /* ** This method is the destructor for completion_cursor objects. */ static int completionDisconnect(sqlite3_vtab *pVtab){ … } /* ** Constructor for a new completion_cursor object. */ static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ … } /* ** Reset the completion_cursor. */ static void completionCursorReset(completion_cursor *pCur){ … } /* ** Destructor for a completion_cursor. */ static int completionClose(sqlite3_vtab_cursor *cur){ … } /* ** Advance a completion_cursor to its next row of output. ** ** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object ** record the current state of the scan. This routine sets ->zCurrentRow ** to the current row of output and then returns. If no more rows remain, ** then ->ePhase is set to COMPLETION_EOF which will signal the virtual ** table that has reached the end of its scan. ** ** The current implementation just lists potential identifiers and ** keywords and filters them by zPrefix. Future enhancements should ** take zLine into account to try to restrict the set of identifiers and ** keywords based on what would be legal at the current point of input. */ static int completionNext(sqlite3_vtab_cursor *cur){ … } /* ** Return values of columns for the row at which the completion_cursor ** is currently pointing. */ static int completionColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ … } /* ** Return the rowid for the current row. In this implementation, the ** rowid is the same as the output value. */ static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ … } /* ** Return TRUE if the cursor has been moved off of the last ** row of output. */ static int completionEof(sqlite3_vtab_cursor *cur){ … } /* ** This method is called to "rewind" the completion_cursor object back ** to the first row of output. This method is always called at least ** once prior to any call to completionColumn() or completionRowid() or ** completionEof(). */ static int completionFilter( sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ … } /* ** SQLite will invoke this method one or more times while planning a query ** that uses the completion virtual table. This routine needs to create ** a query plan for each invocation and compute an estimated cost for that ** plan. ** ** There are two hidden parameters that act as arguments to the table-valued ** function: "prefix" and "wholeline". Bit 0 of idxNum is set if "prefix" ** is available and bit 1 is set if "wholeline" is available. */ static int completionBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ … } /* ** This following structure defines all the methods for the ** completion virtual table. */ static sqlite3_module completionModule = …; #endif /* SQLITE_OMIT_VIRTUALTABLE */ int sqlite3CompletionVtabInit(sqlite3 *db){ … } #ifdef _WIN32 #endif int sqlite3_completion_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/completion.c ********************/ /************************* Begin ../ext/misc/appendvfs.c ******************/ /* ** 2017-10-20 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This file implements a VFS shim that allows an SQLite database to be ** appended onto the end of some other file, such as an executable. ** ** A special record must appear at the end of the file that identifies the ** file as an appended database and provides the offset to the first page ** of the exposed content. (Or, it is the length of the content prefix.) ** For best performance page 1 should be located at a disk page boundary, ** though that is not required. ** ** When opening a database using this VFS, the connection might treat ** the file as an ordinary SQLite database, or it might treat it as a ** database appended onto some other file. The decision is made by ** applying the following rules in order: ** ** (1) An empty file is an ordinary database. ** ** (2) If the file ends with the appendvfs trailer string ** "Start-Of-SQLite3-NNNNNNNN" that file is an appended database. ** ** (3) If the file begins with the standard SQLite prefix string ** "SQLite format 3", that file is an ordinary database. ** ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is ** set, then a new database is appended to the already existing file. ** ** (5) Otherwise, SQLITE_CANTOPEN is returned. ** ** To avoid unnecessary complications with the PENDING_BYTE, the size of ** the file containing the database is limited to 1GiB. (1073741824 bytes) ** This VFS will not read or write past the 1GiB mark. This restriction ** might be lifted in future versions. For now, if you need a larger ** database, then keep it in a separate file. ** ** If the file being opened is a plain database (not an appended one), then ** this shim is a pass-through into the default underlying VFS. (rule 3) **/ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <string.h> #include <assert.h> /* The append mark at the end of the database is: ** ** Start-Of-SQLite3-NNNNNNNN ** 123456789 123456789 12345 ** ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is ** the offset to page 1, and also the length of the prefix content. */ #define APND_MARK_PREFIX … #define APND_MARK_PREFIX_SZ … #define APND_MARK_FOS_SZ … #define APND_MARK_SIZE … /* ** Maximum size of the combined prefix + database + append-mark. This ** must be less than 0x40000000 to avoid locking issues on Windows. */ #define APND_MAX_SIZE … /* ** Try to align the database to an even multiple of APND_ROUNDUP bytes. */ #ifndef APND_ROUNDUP #define APND_ROUNDUP … #endif #define APND_ALIGN_MASK … #define APND_START_ROUNDUP(fsz) … /* ** Forward declaration of objects used by this utility */ ApndVfs; ApndFile; /* Access to a lower-level VFS that (might) implement dynamic loading, ** access to randomness, etc. */ #define ORIGVFS(p) … #define ORIGFILE(p) … /* An open appendvfs file ** ** An instance of this structure describes the appended database file. ** A separate sqlite3_file object is always appended. The appended ** sqlite3_file object (which can be accessed using ORIGFILE()) describes ** the entire file, including the prefix, the database, and the ** append-mark. ** ** The structure of an AppendVFS database is like this: ** ** +-------------+---------+----------+-------------+ ** | prefix-file | padding | database | append-mark | ** +-------------+---------+----------+-------------+ ** ^ ^ ** | | ** iPgOne iMark ** ** ** "prefix file" - file onto which the database has been appended. ** "padding" - zero or more bytes inserted so that "database" ** starts on an APND_ROUNDUP boundary ** "database" - The SQLite database file ** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates ** the offset from the start of prefix-file to the start ** of "database". ** ** The size of the database is iMark - iPgOne. ** ** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value ** of iPgOne stored as a big-ending 64-bit integer. ** ** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). ** Or, iMark is -1 to indicate that it has not yet been written. */ struct ApndFile { … }; /* ** Methods for ApndFile */ static int apndClose(sqlite3_file*); static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); static int apndTruncate(sqlite3_file*, sqlite3_int64 size); static int apndSync(sqlite3_file*, int flags); static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); static int apndLock(sqlite3_file*, int); static int apndUnlock(sqlite3_file*, int); static int apndCheckReservedLock(sqlite3_file*, int *pResOut); static int apndFileControl(sqlite3_file*, int op, void *pArg); static int apndSectorSize(sqlite3_file*); static int apndDeviceCharacteristics(sqlite3_file*); static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); static int apndShmLock(sqlite3_file*, int offset, int n, int flags); static void apndShmBarrier(sqlite3_file*); static int apndShmUnmap(sqlite3_file*, int deleteFlag); static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); /* ** Methods for ApndVfs */ static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); static void apndDlClose(sqlite3_vfs*, void*); static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); static int apndSleep(sqlite3_vfs*, int microseconds); static int apndCurrentTime(sqlite3_vfs*, double*); static int apndGetLastError(sqlite3_vfs*, int, char *); static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); static sqlite3_vfs apnd_vfs = …; static const sqlite3_io_methods apnd_io_methods = …; /* ** Close an apnd-file. */ static int apndClose(sqlite3_file *pFile){ … } /* ** Read data from an apnd-file. */ static int apndRead( sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst ){ … } /* ** Add the append-mark onto what should become the end of the file. * If and only if this succeeds, internal ApndFile.iMark is updated. * Parameter iWriteEnd is the appendvfs-relative offset of the new mark. */ static int apndWriteMark( ApndFile *paf, sqlite3_file *pFile, sqlite_int64 iWriteEnd ){ … } /* ** Write data to an apnd-file. */ static int apndWrite( sqlite3_file *pFile, const void *zBuf, int iAmt, sqlite_int64 iOfst ){ … } /* ** Truncate an apnd-file. */ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ … } /* ** Sync an apnd-file. */ static int apndSync(sqlite3_file *pFile, int flags){ … } /* ** Return the current file-size of an apnd-file. ** If the append mark is not yet there, the file-size is 0. */ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ … } /* ** Lock an apnd-file. */ static int apndLock(sqlite3_file *pFile, int eLock){ … } /* ** Unlock an apnd-file. */ static int apndUnlock(sqlite3_file *pFile, int eLock){ … } /* ** Check if another file-handle holds a RESERVED lock on an apnd-file. */ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ … } /* ** File control method. For custom operations on an apnd-file. */ static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ … } /* ** Return the sector-size in bytes for an apnd-file. */ static int apndSectorSize(sqlite3_file *pFile){ … } /* ** Return the device characteristic flags supported by an apnd-file. */ static int apndDeviceCharacteristics(sqlite3_file *pFile){ … } /* Create a shared memory file mapping */ static int apndShmMap( sqlite3_file *pFile, int iPg, int pgsz, int bExtend, void volatile **pp ){ … } /* Perform locking on a shared-memory segment */ static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ … } /* Memory barrier operation on shared memory */ static void apndShmBarrier(sqlite3_file *pFile){ … } /* Unmap a shared memory segment */ static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ … } /* Fetch a page of a memory-mapped file */ static int apndFetch( sqlite3_file *pFile, sqlite3_int64 iOfst, int iAmt, void **pp ){ … } /* Release a memory-mapped page */ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ … } /* ** Try to read the append-mark off the end of a file. Return the ** start of the appended database if the append-mark is present. ** If there is no valid append-mark, return -1; ** ** An append-mark is only valid if the NNNNNNNN start-of-database offset ** indicates that the appended database contains at least one page. The ** start-of-database value must be a multiple of 512. */ static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ … } static const char apvfsSqliteHdr[] = …; /* ** Check to see if the file is an appendvfs SQLite database file. ** Return true iff it is such. Parameter sz is the file's size. */ static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ … } /* ** Check to see if the file is an ordinary SQLite database file. ** Return true iff so. Parameter sz is the file's size. */ static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ … } /* ** Open an apnd file handle. */ static int apndOpen( sqlite3_vfs *pApndVfs, const char *zName, sqlite3_file *pFile, int flags, int *pOutFlags ){ … } /* ** Delete an apnd file. ** For an appendvfs, this could mean delete the appendvfs portion, ** leaving the appendee as it was before it gained an appendvfs. ** For now, this code deletes the underlying file too. */ static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ … } /* ** All other VFS methods are pass-thrus. */ static int apndAccess( sqlite3_vfs *pVfs, const char *zPath, int flags, int *pResOut ){ … } static int apndFullPathname( sqlite3_vfs *pVfs, const char *zPath, int nOut, char *zOut ){ … } static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ … } static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ … } static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ … } static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ … } static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ … } static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ … } static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ … } static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ … } static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ … } static int apndSetSystemCall( sqlite3_vfs *pVfs, const char *zName, sqlite3_syscall_ptr pCall ){ … } static sqlite3_syscall_ptr apndGetSystemCall( sqlite3_vfs *pVfs, const char *zName ){ … } static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ … } #ifdef _WIN32 #endif /* ** This routine is called when the extension is loaded. ** Register the new VFS. */ int sqlite3_appendvfs_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ … } /************************* End ../ext/misc/appendvfs.c ********************/ #endif #ifdef SQLITE_HAVE_ZLIB /************************* Begin ../ext/misc/zipfile.c ******************/ /* ** 2017-12-26 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This file implements a virtual table for reading and writing ZIP archive ** files. ** ** Usage example: ** ** SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename); ** ** Current limitations: ** ** * No support for encryption ** * No support for ZIP archives spanning multiple files ** * No support for zip64 extensions ** * Only the "inflate/deflate" (zlib) compression method is supported */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <stdio.h> #include <string.h> #include <assert.h> #include <stdint.h> #include <zlib.h> #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION #ifndef UINT32_TYPE # ifdef HAVE_UINT32_T #define UINT32_TYPE … # else #define UINT32_TYPE … # endif #endif #ifndef UINT16_TYPE # ifdef HAVE_UINT16_T #define UINT16_TYPE … # else #define UINT16_TYPE … # endif #endif /* typedef sqlite3_int64 i64; */ /* typedef unsigned char u8; */ /* typedef UINT32_TYPE u32; // 4-byte unsigned integer // */ /* typedef UINT16_TYPE u16; // 2-byte unsigned integer // */ #define MIN … #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) #define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS … #endif #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) #define ALWAYS … #define NEVER … #elif !defined(NDEBUG) #define ALWAYS … #define NEVER … #else #define ALWAYS … #define NEVER … #endif #endif /* SQLITE_AMALGAMATION */ /* ** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. ** ** In some ways it would be better to obtain these values from system ** header files. But, the dependency is undesirable and (a) these ** have been stable for decades, (b) the values are part of POSIX and ** are also made explicit in [man stat], and (c) are part of the ** file format for zip archives. */ #ifndef S_IFDIR #define S_IFDIR … #endif #ifndef S_IFREG #define S_IFREG … #endif #ifndef S_IFLNK #define S_IFLNK … #endif static const char ZIPFILE_SCHEMA[] = "CREATE TABLE y(" "name PRIMARY KEY," /* 0: Name of file in zip archive */ "mode," /* 1: POSIX mode for file */ "mtime," /* 2: Last modification time (secs since 1970)*/ "sz," /* 3: Size of object */ "rawdata," /* 4: Raw data */ "data," /* 5: Uncompressed data */ "method," /* 6: Compression method (integer) */ "z HIDDEN" /* 7: Name of zip file */ ") WITHOUT ROWID;"; #define ZIPFILE_F_COLUMN_IDX … #define ZIPFILE_BUFFER_SIZE … /* ** Magic numbers used to read and write zip files. ** ** ZIPFILE_NEWENTRY_MADEBY: ** Use this value for the "version-made-by" field in new zip file ** entries. The upper byte indicates "unix", and the lower byte ** indicates that the zip file matches pkzip specification 3.0. ** This is what info-zip seems to do. ** ** ZIPFILE_NEWENTRY_REQUIRED: ** Value for "version-required-to-extract" field of new entries. ** Version 2.0 is required to support folders and deflate compression. ** ** ZIPFILE_NEWENTRY_FLAGS: ** Value for "general-purpose-bit-flags" field of new entries. Bit ** 11 means "utf-8 filename and comment". ** ** ZIPFILE_SIGNATURE_CDS: ** First 4 bytes of a valid CDS record. ** ** ZIPFILE_SIGNATURE_LFH: ** First 4 bytes of a valid LFH record. ** ** ZIPFILE_SIGNATURE_EOCD ** First 4 bytes of a valid EOCD record. */ #define ZIPFILE_EXTRA_TIMESTAMP … #define ZIPFILE_NEWENTRY_MADEBY … #define ZIPFILE_NEWENTRY_REQUIRED … #define ZIPFILE_NEWENTRY_FLAGS … #define ZIPFILE_SIGNATURE_CDS … #define ZIPFILE_SIGNATURE_LFH … #define ZIPFILE_SIGNATURE_EOCD … /* ** The sizes of the fixed-size part of each of the three main data ** structures in a zip archive. */ #define ZIPFILE_LFH_FIXED_SZ … #define ZIPFILE_EOCD_FIXED_SZ … #define ZIPFILE_CDS_FIXED_SZ … /* *** 4.3.16 End of central directory record: *** *** end of central dir signature 4 bytes (0x06054b50) *** number of this disk 2 bytes *** number of the disk with the *** start of the central directory 2 bytes *** total number of entries in the *** central directory on this disk 2 bytes *** total number of entries in *** the central directory 2 bytes *** size of the central directory 4 bytes *** offset of start of central *** directory with respect to *** the starting disk number 4 bytes *** .ZIP file comment length 2 bytes *** .ZIP file comment (variable size) */ typedef struct ZipfileEOCD ZipfileEOCD; struct ZipfileEOCD { u16 iDisk; u16 iFirstDisk; u16 nEntry; u16 nEntryTotal; u32 nSize; u32 iOffset; }; /* *** 4.3.12 Central directory structure: *** *** ... *** *** central file header signature 4 bytes (0x02014b50) *** version made by 2 bytes *** version needed to extract 2 bytes *** general purpose bit flag 2 bytes *** compression method 2 bytes *** last mod file time 2 bytes *** last mod file date 2 bytes *** crc-32 4 bytes *** compressed size 4 bytes *** uncompressed size 4 bytes *** file name length 2 bytes *** extra field length 2 bytes *** file comment length 2 bytes *** disk number start 2 bytes *** internal file attributes 2 bytes *** external file attributes 4 bytes *** relative offset of local header 4 bytes */ typedef struct ZipfileCDS ZipfileCDS; struct ZipfileCDS { u16 iVersionMadeBy; u16 iVersionExtract; u16 flags; u16 iCompression; u16 mTime; u16 mDate; u32 crc32; u32 szCompressed; u32 szUncompressed; u16 nFile; u16 nExtra; u16 nComment; u16 iDiskStart; u16 iInternalAttr; u32 iExternalAttr; u32 iOffset; char *zFile; /* Filename (sqlite3_malloc()) */ }; /* *** 4.3.7 Local file header: *** *** local file header signature 4 bytes (0x04034b50) *** version needed to extract 2 bytes *** general purpose bit flag 2 bytes *** compression method 2 bytes *** last mod file time 2 bytes *** last mod file date 2 bytes *** crc-32 4 bytes *** compressed size 4 bytes *** uncompressed size 4 bytes *** file name length 2 bytes *** extra field length 2 bytes *** */ typedef struct ZipfileLFH ZipfileLFH; struct ZipfileLFH { u16 iVersionExtract; u16 flags; u16 iCompression; u16 mTime; u16 mDate; u32 crc32; u32 szCompressed; u32 szUncompressed; u16 nFile; u16 nExtra; }; typedef struct ZipfileEntry ZipfileEntry; struct ZipfileEntry { ZipfileCDS cds; /* Parsed CDS record */ u32 mUnixTime; /* Modification time, in UNIX format */ u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ i64 iDataOff; /* Offset to data in file (if aData==0) */ u8 *aData; /* cds.szCompressed bytes of compressed data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; /* ** Cursor type for zipfile tables. */ typedef struct ZipfileCsr ZipfileCsr; struct ZipfileCsr { sqlite3_vtab_cursor base; /* Base class - must be first */ i64 iId; /* Cursor ID */ u8 bEof; /* True when at EOF */ u8 bNoop; /* If next xNext() call is no-op */ /* Used outside of write transactions */ FILE *pFile; /* Zip file */ i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ ZipfileEntry *pCurrent; /* Current entry */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ char *zFile; /* Zip file this table accesses (may be NULL) */ sqlite3 *db; /* Host database connection */ u8 *aBuffer; /* Temporary buffer used for various tasks */ ZipfileCsr *pCsrList; /* List of cursors */ i64 iNextCsrid; /* The following are used by write transactions only */ ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */ ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */ FILE *pWriteFd; /* File handle open on zip archive */ i64 szCurrent; /* Current size of zip archive */ i64 szOrig; /* Size of archive at start of transaction */ }; /* ** Set the error message contained in context ctx to the results of ** vprintf(zFmt, ...). */ static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ char *zMsg = 0; va_list ap; va_start(ap, zFmt); zMsg = sqlite3_vmprintf(zFmt, ap); sqlite3_result_error(ctx, zMsg, -1); sqlite3_free(zMsg); va_end(ap); } /* ** If string zIn is quoted, dequote it in place. Otherwise, if the string ** is not quoted, do nothing. */ static void zipfileDequote(char *zIn){ char q = zIn[0]; if( q=='"' || q=='\'' || q=='`' || q=='[' ){ int iIn = 1; int iOut = 0; if( q=='[' ) q = ']'; while( ALWAYS(zIn[iIn]) ){ char c = zIn[iIn++]; if( c==q && zIn[iIn++]!=q ) break; zIn[iOut++] = c; } zIn[iOut] = '\0'; } } /* ** Construct a new ZipfileTab virtual table object. ** ** argv[0] -> module name ("zipfile") ** argv[1] -> database name ** argv[2] -> table name ** argv[...] -> "column name" and other module argument fields. */ static int zipfileConnect( sqlite3 *db, void *pAux, int argc, const char *const*argv, sqlite3_vtab **ppVtab, char **pzErr ){ int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE; int nFile = 0; const char *zFile = 0; ZipfileTab *pNew = 0; int rc; (void)pAux; /* If the table name is not "zipfile", require that the argument be ** specified. This stops zipfile tables from being created as: ** ** CREATE VIRTUAL TABLE zzz USING zipfile(); ** ** It does not prevent: ** ** CREATE VIRTUAL TABLE zipfile USING zipfile(); */ assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); return SQLITE_ERROR; } if( argc>3 ){ zFile = argv[3]; nFile = (int)strlen(zFile)+1; } rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); if( rc==SQLITE_OK ){ pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte+nFile); pNew->db = db; pNew->aBuffer = (u8*)&pNew[1]; if( zFile ){ pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE]; memcpy(pNew->zFile, zFile, nFile); zipfileDequote(pNew->zFile); } } sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); *ppVtab = (sqlite3_vtab*)pNew; return rc; } /* ** Free the ZipfileEntry structure indicated by the only argument. */ static void zipfileEntryFree(ZipfileEntry *p){ if( p ){ sqlite3_free(p->cds.zFile); sqlite3_free(p); } } /* ** Release resources that should be freed at the end of a write ** transaction. */ static void zipfileCleanupTransaction(ZipfileTab *pTab){ ZipfileEntry *pEntry; ZipfileEntry *pNext; if( pTab->pWriteFd ){ fclose(pTab->pWriteFd); pTab->pWriteFd = 0; } for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ pNext = pEntry->pNext; zipfileEntryFree(pEntry); } pTab->pFirstEntry = 0; pTab->pLastEntry = 0; pTab->szCurrent = 0; pTab->szOrig = 0; } /* ** This method is the destructor for zipfile vtab objects. */ static int zipfileDisconnect(sqlite3_vtab *pVtab){ zipfileCleanupTransaction((ZipfileTab*)pVtab); sqlite3_free(pVtab); return SQLITE_OK; } /* ** Constructor for a new ZipfileCsr object. */ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ZipfileTab *pTab = (ZipfileTab*)p; ZipfileCsr *pCsr; pCsr = sqlite3_malloc(sizeof(*pCsr)); *ppCsr = (sqlite3_vtab_cursor*)pCsr; if( pCsr==0 ){ return SQLITE_NOMEM; } memset(pCsr, 0, sizeof(*pCsr)); pCsr->iId = ++pTab->iNextCsrid; pCsr->pCsrNext = pTab->pCsrList; pTab->pCsrList = pCsr; return SQLITE_OK; } /* ** Reset a cursor back to the state it was in when first returned ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ ZipfileEntry *p; ZipfileEntry *pNext; pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); pCsr->pFile = 0; zipfileEntryFree(pCsr->pCurrent); pCsr->pCurrent = 0; } for(p=pCsr->pFreeEntry; p; p=pNext){ pNext = p->pNext; zipfileEntryFree(p); } } /* ** Destructor for an ZipfileCsr. */ static int zipfileClose(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab); ZipfileCsr **pp; zipfileResetCursor(pCsr); /* Remove this cursor from the ZipfileTab.pCsrList list. */ for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); *pp = pCsr->pCsrNext; sqlite3_free(pCsr); return SQLITE_OK; } /* ** Set the error message for the virtual table associated with cursor ** pCsr to the results of vprintf(zFmt, ...). */ static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); sqlite3_free(pTab->base.zErrMsg); pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap); va_end(ap); } static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); sqlite3_free(pCsr->base.pVtab->zErrMsg); pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); va_end(ap); } /* ** Read nRead bytes of data from offset iOff of file pFile into buffer ** aRead[]. Return SQLITE_OK if successful, or an SQLite error code ** otherwise. ** ** If an error does occur, output variable (*pzErrmsg) may be set to point ** to an English language error message. It is the responsibility of the ** caller to eventually free this buffer using ** sqlite3_free(). */ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ int nRead, /* Number of bytes to read */ i64 iOff, /* Offset to read from */ char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ ){ size_t n; fseek(pFile, (long)iOff, SEEK_SET); n = fread(aRead, 1, nRead, pFile); if( (int)n!=nRead ){ *pzErrmsg = sqlite3_mprintf("error in fread()"); return SQLITE_ERROR; } return SQLITE_OK; } static int zipfileAppendData( ZipfileTab *pTab, const u8 *aWrite, int nWrite ){ if( nWrite>0 ){ size_t n = nWrite; fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); if( (int)n!=nWrite ){ pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); return SQLITE_ERROR; } pTab->szCurrent += nWrite; } return SQLITE_OK; } /* ** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. */ static u16 zipfileGetU16(const u8 *aBuf){ return (aBuf[1] << 8) + aBuf[0]; } /* ** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. */ static u32 zipfileGetU32(const u8 *aBuf){ if( aBuf==0 ) return 0; return ((u32)(aBuf[3]) << 24) + ((u32)(aBuf[2]) << 16) + ((u32)(aBuf[1]) << 8) + ((u32)(aBuf[0]) << 0); } /* ** Write a 16-bit little endiate integer into buffer aBuf. */ static void zipfilePutU16(u8 *aBuf, u16 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; } /* ** Write a 32-bit little endiate integer into buffer aBuf. */ static void zipfilePutU32(u8 *aBuf, u32 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; aBuf[2] = (val>>16) & 0xFF; aBuf[3] = (val>>24) & 0xFF; } #define zipfileRead32 … #define zipfileRead16 … #define zipfileWrite32 … #define zipfileWrite16 … /* ** Magic numbers used to read CDS records. */ #define ZIPFILE_CDS_NFILE_OFF … #define ZIPFILE_CDS_SZCOMPRESSED_OFF … /* ** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR ** if the record is not well-formed, or SQLITE_OK otherwise. */ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ u8 *aRead = aBuf; u32 sig = zipfileRead32(aRead); int rc = SQLITE_OK; if( sig!=ZIPFILE_SIGNATURE_CDS ){ rc = SQLITE_ERROR; }else{ pCDS->iVersionMadeBy = zipfileRead16(aRead); pCDS->iVersionExtract = zipfileRead16(aRead); pCDS->flags = zipfileRead16(aRead); pCDS->iCompression = zipfileRead16(aRead); pCDS->mTime = zipfileRead16(aRead); pCDS->mDate = zipfileRead16(aRead); pCDS->crc32 = zipfileRead32(aRead); pCDS->szCompressed = zipfileRead32(aRead); pCDS->szUncompressed = zipfileRead32(aRead); assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); pCDS->nFile = zipfileRead16(aRead); pCDS->nExtra = zipfileRead16(aRead); pCDS->nComment = zipfileRead16(aRead); pCDS->iDiskStart = zipfileRead16(aRead); pCDS->iInternalAttr = zipfileRead16(aRead); pCDS->iExternalAttr = zipfileRead32(aRead); pCDS->iOffset = zipfileRead32(aRead); assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] ); } return rc; } /* ** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR ** if the record is not well-formed, or SQLITE_OK otherwise. */ static int zipfileReadLFH( u8 *aBuffer, ZipfileLFH *pLFH ){ u8 *aRead = aBuffer; int rc = SQLITE_OK; u32 sig = zipfileRead32(aRead); if( sig!=ZIPFILE_SIGNATURE_LFH ){ rc = SQLITE_ERROR; }else{ pLFH->iVersionExtract = zipfileRead16(aRead); pLFH->flags = zipfileRead16(aRead); pLFH->iCompression = zipfileRead16(aRead); pLFH->mTime = zipfileRead16(aRead); pLFH->mDate = zipfileRead16(aRead); pLFH->crc32 = zipfileRead32(aRead); pLFH->szCompressed = zipfileRead32(aRead); pLFH->szUncompressed = zipfileRead32(aRead); pLFH->nFile = zipfileRead16(aRead); pLFH->nExtra = zipfileRead16(aRead); } return rc; } /* ** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. ** Scan through this buffer to find an "extra-timestamp" field. If one ** exists, extract the 32-bit modification-timestamp from it and store ** the value in output parameter *pmTime. ** ** Zero is returned if no extra-timestamp record could be found (and so ** *pmTime is left unchanged), or non-zero otherwise. ** ** The general format of an extra field is: ** ** Header ID 2 bytes ** Data Size 2 bytes ** Data N bytes */ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ int ret = 0; u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; while( p<pEnd ){ u16 id = zipfileRead16(p); u16 nByte = zipfileRead16(p); switch( id ){ case ZIPFILE_EXTRA_TIMESTAMP: { u8 b = p[0]; if( b & 0x01 ){ /* 0x01 -> modtime is present */ *pmTime = zipfileGetU32(&p[1]); ret = 1; } break; } } p += nByte; } return ret; } /* ** Convert the standard MS-DOS timestamp stored in the mTime and mDate ** fields of the CDS structure passed as the only argument to a 32-bit ** UNIX seconds-since-the-epoch timestamp. Return the result. ** ** "Standard" MS-DOS time format: ** ** File modification time: ** Bits 00-04: seconds divided by 2 ** Bits 05-10: minute ** Bits 11-15: hour ** File modification date: ** Bits 00-04: day ** Bits 05-08: month (1-12) ** Bits 09-15: years from 1980 ** ** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx */ static u32 zipfileMtime(ZipfileCDS *pCDS){ int Y,M,D,X1,X2,A,B,sec,min,hr; i64 JDsec; Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); M = ((pCDS->mDate >> 5) & 0x0F); D = (pCDS->mDate & 0x1F); sec = (pCDS->mTime & 0x1F)*2; min = (pCDS->mTime >> 5) & 0x3F; hr = (pCDS->mTime >> 11) & 0x1F; if( M<=2 ){ Y--; M += 12; } X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; A = Y/100; B = 2 - A + (A/4); JDsec = (i64)((X1 + X2 + D + B - 1524.5)*86400) + hr*3600 + min*60 + sec; return (u32)(JDsec - (i64)24405875*(i64)8640); } /* ** The opposite of zipfileMtime(). This function populates the mTime and ** mDate fields of the CDS structure passed as the first argument according ** to the UNIX timestamp value passed as the second. */ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ i64 JD = (i64)2440588 + mUnixTime / (24*60*60); int A, B, C, D, E; int yr, mon, day; int hr, min, sec; A = (int)((JD - 1867216.25)/36524.25); A = (int)(JD + 1 + A - (A/4)); B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; E = (int)((B-D)/30.6001); day = B - D - (int)(30.6001*E); mon = (E<14 ? E-1 : E-13); yr = mon>2 ? C-4716 : C-4715; hr = (mUnixTime % (24*60*60)) / (60*60); min = (mUnixTime % (60*60)) / 60; sec = (mUnixTime % 60); if( yr>=1980 ){ pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); }else{ pCds->mDate = pCds->mTime = 0; } assert( mUnixTime<315507600 || mUnixTime==zipfileMtime(pCds) || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) /* || (mUnixTime % 2) */ ); } /* ** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in ** size) containing an entire zip archive image. Or, if aBlob is NULL, ** then pFile is a file-handle open on a zip file. In either case, this ** function creates a ZipfileEntry object based on the zip archive entry ** for which the CDS record is at offset iOff. ** ** If successful, SQLITE_OK is returned and (*ppEntry) set to point to ** the new object. Otherwise, an SQLite error code is returned and the ** final value of (*ppEntry) undefined. */ static int zipfileGetEntry( ZipfileTab *pTab, /* Store any error message here */ const u8 *aBlob, /* Pointer to in-memory file image */ int nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* If aBlob==0, read from this file */ i64 iOff, /* Offset of CDS record */ ZipfileEntry **ppEntry /* OUT: Pointer to new object */ ){ u8 *aRead; char **pzErr = &pTab->base.zErrMsg; int rc = SQLITE_OK; (void)nBlob; if( aBlob==0 ){ aRead = pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ aRead = (u8*)&aBlob[iOff]; } if( rc==SQLITE_OK ){ sqlite3_int64 nAlloc; ZipfileEntry *pNew; int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); nAlloc = sizeof(ZipfileEntry) + nExtra; if( aBlob ){ nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); } pNew = (ZipfileEntry*)sqlite3_malloc64(nAlloc); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ memset(pNew, 0, sizeof(ZipfileEntry)); rc = zipfileReadCDS(aRead, &pNew->cds); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); }else if( aBlob==0 ){ rc = zipfileReadData( pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr ); }else{ aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; } } if( rc==SQLITE_OK ){ u32 *pt = &pNew->mUnixTime; pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); pNew->aExtra = (u8*)&pNew[1]; memcpy(pNew->aExtra, &aRead[nFile], nExtra); if( pNew->cds.zFile==0 ){ rc = SQLITE_NOMEM; }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ pNew->mUnixTime = zipfileMtime(&pNew->cds); } } if( rc==SQLITE_OK ){ static const int szFix = ZIPFILE_LFH_FIXED_SZ; ZipfileLFH lfh; if( pFile ){ rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); }else{ aRead = (u8*)&aBlob[pNew->cds.iOffset]; } if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); if( rc==SQLITE_OK ){ pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; pNew->iDataOff += lfh.nFile + lfh.nExtra; if( aBlob && pNew->cds.szCompressed ){ pNew->aData = &pNew->aExtra[nExtra]; memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); } }else{ *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)pNew->cds.iOffset ); } } if( rc!=SQLITE_OK ){ zipfileEntryFree(pNew); }else{ *ppEntry = pNew; } } return rc; } /* ** Advance an ZipfileCsr to its next row of output. */ static int zipfileNext(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; int rc = SQLITE_OK; if( pCsr->pFile ){ i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; zipfileEntryFree(pCsr->pCurrent); pCsr->pCurrent = 0; if( pCsr->iNextOff>=iEof ){ pCsr->bEof = 1; }else{ ZipfileEntry *p = 0; ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); if( rc==SQLITE_OK ){ pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; } pCsr->pCurrent = p; } }else{ if( !pCsr->bNoop ){ pCsr->pCurrent = pCsr->pCurrent->pNext; } if( pCsr->pCurrent==0 ){ pCsr->bEof = 1; } } pCsr->bNoop = 0; return rc; } static void zipfileFree(void *p) { sqlite3_free(p); } /* ** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the ** size is nOut bytes. This function uncompresses the data and sets the ** return value in context pCtx to the result (a blob). ** ** If an error occurs, an error code is left in pCtx instead. */ static void zipfileInflate( sqlite3_context *pCtx, /* Store result here */ const u8 *aIn, /* Compressed data */ int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ ){ u8 *aRes = sqlite3_malloc(nOut); if( aRes==0 ){ sqlite3_result_error_nomem(pCtx); }else{ int err; z_stream str; memset(&str, 0, sizeof(str)); str.next_in = (Byte*)aIn; str.avail_in = nIn; str.next_out = (Byte*)aRes; str.avail_out = nOut; err = inflateInit2(&str, -15); if( err!=Z_OK ){ zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err); }else{ err = inflate(&str, Z_NO_FLUSH); if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); aRes = 0; } } sqlite3_free(aRes); inflateEnd(&str); } } /* ** Buffer aIn (size nIn bytes) contains uncompressed data. This function ** compresses it and sets (*ppOut) to point to a buffer containing the ** compressed data. The caller is responsible for eventually calling ** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) ** is set to the size of buffer (*ppOut) in bytes. ** ** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error ** code is returned and an error message left in virtual-table handle ** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this ** case. */ static int zipfileDeflate( const u8 *aIn, int nIn, /* Input */ u8 **ppOut, int *pnOut, /* Output */ char **pzErr /* OUT: Error message */ ){ int rc = SQLITE_OK; sqlite3_int64 nAlloc; z_stream str; u8 *aOut; memset(&str, 0, sizeof(str)); str.next_in = (Bytef*)aIn; str.avail_in = nIn; deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); nAlloc = deflateBound(&str, nIn); aOut = (u8*)sqlite3_malloc64(nAlloc); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ int res; str.next_out = aOut; str.avail_out = nAlloc; res = deflate(&str, Z_FINISH); if( res==Z_STREAM_END ){ *ppOut = aOut; *pnOut = (int)str.total_out; }else{ sqlite3_free(aOut); *pzErr = sqlite3_mprintf("zipfile: deflate() error"); rc = SQLITE_ERROR; } deflateEnd(&str); } return rc; } /* ** Return values of columns for the row at which the series_cursor ** is currently pointing. */ static int zipfileColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ int i /* Which column to return */ ){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; ZipfileCDS *pCDS = &pCsr->pCurrent->cds; int rc = SQLITE_OK; switch( i ){ case 0: /* name */ sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); break; case 1: /* mode */ /* TODO: Whether or not the following is correct surely depends on ** the platform on which the archive was created. */ sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); break; case 2: { /* mtime */ sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); break; } case 3: { /* sz */ if( sqlite3_vtab_nochange(ctx)==0 ){ sqlite3_result_int64(ctx, pCDS->szUncompressed); } break; } case 4: /* rawdata */ if( sqlite3_vtab_nochange(ctx) ) break; case 5: { /* data */ if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ int sz = pCDS->szCompressed; int szFinal = pCDS->szUncompressed; if( szFinal>0 ){ u8 *aBuf; u8 *aFree = 0; if( pCsr->pCurrent->aData ){ aBuf = pCsr->pCurrent->aData; }else{ aBuf = aFree = sqlite3_malloc64(sz); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ FILE *pFile = pCsr->pFile; if( pFile==0 ){ pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; } rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, &pCsr->base.pVtab->zErrMsg ); } } if( rc==SQLITE_OK ){ if( i==5 && pCDS->iCompression ){ zipfileInflate(ctx, aBuf, sz, szFinal); }else{ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); } } sqlite3_free(aFree); }else{ /* Figure out if this is a directory or a zero-sized file. Consider ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ u32 mode = pCDS->iExternalAttr >> 16; if( !(mode & S_IFDIR) && pCDS->nFile>=1 && pCDS->zFile[pCDS->nFile-1]!='/' ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } } break; } case 6: /* method */ sqlite3_result_int(ctx, pCDS->iCompression); break; default: /* z */ assert( i==7 ); sqlite3_result_int64(ctx, pCsr->iId); break; } return rc; } /* ** Return TRUE if the cursor is at EOF. */ static int zipfileEof(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; return pCsr->bEof; } /* ** If aBlob is not NULL, then it points to a buffer nBlob bytes in size ** containing an entire zip archive image. Or, if aBlob is NULL, then pFile ** is guaranteed to be a file-handle open on a zip file. ** ** This function attempts to locate the EOCD record within the zip archive ** and populate *pEOCD with the results of decoding it. SQLITE_OK is ** returned if successful. Otherwise, an SQLite error code is returned and ** an English language error message may be left in virtual-table pTab. */ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ const u8 *aBlob, /* Pointer to in-memory file image */ int nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ int nRead; /* Bytes to read from file */ int rc = SQLITE_OK; memset(pEOCD, 0, sizeof(ZipfileEOCD)); if( aBlob==0 ){ i64 iOff; /* Offset to read from */ i64 szFile; /* Total size of file in bytes */ fseek(pFile, 0, SEEK_END); szFile = (i64)ftell(pFile); if( szFile==0 ){ return SQLITE_OK; } nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); iOff = szFile - nRead; rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); }else{ nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); aRead = (u8*)&aBlob[nBlob-nRead]; } if( rc==SQLITE_OK ){ int i; /* Scan backwards looking for the signature bytes */ for(i=nRead-20; i>=0; i--){ if( aRead[i]==0x50 && aRead[i+1]==0x4b && aRead[i+2]==0x05 && aRead[i+3]==0x06 ){ break; } } if( i<0 ){ pTab->base.zErrMsg = sqlite3_mprintf( "cannot find end of central directory record" ); return SQLITE_ERROR; } aRead += i+4; pEOCD->iDisk = zipfileRead16(aRead); pEOCD->iFirstDisk = zipfileRead16(aRead); pEOCD->nEntry = zipfileRead16(aRead); pEOCD->nEntryTotal = zipfileRead16(aRead); pEOCD->nSize = zipfileRead32(aRead); pEOCD->iOffset = zipfileRead32(aRead); } return rc; } /* ** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry ** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added ** to the end of the list. Otherwise, it is added to the list immediately ** before pBefore (which is guaranteed to be a part of said list). */ static void zipfileAddEntry( ZipfileTab *pTab, ZipfileEntry *pBefore, ZipfileEntry *pNew ){ assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); assert( pNew->pNext==0 ); if( pBefore==0 ){ if( pTab->pFirstEntry==0 ){ pTab->pFirstEntry = pTab->pLastEntry = pNew; }else{ assert( pTab->pLastEntry->pNext==0 ); pTab->pLastEntry->pNext = pNew; pTab->pLastEntry = pNew; } }else{ ZipfileEntry **pp; for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); pNew->pNext = pBefore; *pp = pNew; } } static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ ZipfileEOCD eocd; int rc; int i; i64 iOff; rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); iOff = eocd.iOffset; for(i=0; rc==SQLITE_OK && i<eocd.nEntry; i++){ ZipfileEntry *pNew = 0; rc = zipfileGetEntry(pTab, aBlob, nBlob, pTab->pWriteFd, iOff, &pNew); if( rc==SQLITE_OK ){ zipfileAddEntry(pTab, 0, pNew); iOff += ZIPFILE_CDS_FIXED_SZ; iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; } } return rc; } /* ** xFilter callback. */ static int zipfileFilter( sqlite3_vtab_cursor *cur, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; const char *zFile = 0; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ int bInMemory = 0; /* True for an in-memory zipfile */ (void)idxStr; (void)argc; zipfileResetCursor(pCsr); if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ zipfileCursorErr(pCsr, "zipfile() function requires an argument"); return SQLITE_ERROR; }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ static const u8 aEmptyBlob = 0; const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); int nBlob = sqlite3_value_bytes(argv[0]); assert( pTab->pFirstEntry==0 ); if( aBlob==0 ){ aBlob = &aEmptyBlob; nBlob = 0; } rc = zipfileLoadDirectory(pTab, aBlob, nBlob); pCsr->pFreeEntry = pTab->pFirstEntry; pTab->pFirstEntry = pTab->pLastEntry = 0; if( rc!=SQLITE_OK ) return rc; bInMemory = 1; }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } if( 0==pTab->pWriteFd && 0==bInMemory ){ pCsr->pFile = zFile ? fopen(zFile, "rb") : 0; if( pCsr->pFile==0 ){ zipfileCursorErr(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); if( rc==SQLITE_OK ){ if( pCsr->eocd.nEntry==0 ){ pCsr->bEof = 1; }else{ pCsr->iNextOff = pCsr->eocd.iOffset; rc = zipfileNext(cur); } } } }else{ pCsr->bNoop = 1; pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; rc = zipfileNext(cur); } return rc; } /* ** xBestIndex callback. */ static int zipfileBestIndex( sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo ){ int i; int idx = -1; int unusable = 0; (void)tab; for(i=0; i<pIdxInfo->nConstraint; i++){ const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue; if( pCons->usable==0 ){ unusable = 1; }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){ idx = i; } } pIdxInfo->estimatedCost = 1000.0; if( idx>=0 ){ pIdxInfo->aConstraintUsage[idx].argvIndex = 1; pIdxInfo->aConstraintUsage[idx].omit = 1; pIdxInfo->idxNum = 1; }else if( unusable ){ return SQLITE_CONSTRAINT; } return SQLITE_OK; } static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; pNew = sqlite3_malloc(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); pNew->cds.zFile = sqlite3_mprintf("%s", zPath); if( pNew->cds.zFile==0 ){ sqlite3_free(pNew); pNew = 0; } } return pNew; } static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ ZipfileCDS *pCds = &pEntry->cds; u8 *a = aBuf; pCds->nExtra = 9; /* Write the LFH itself */ zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); zipfileWrite16(a, pCds->iVersionExtract); zipfileWrite16(a, pCds->flags); zipfileWrite16(a, pCds->iCompression); zipfileWrite16(a, pCds->mTime); zipfileWrite16(a, pCds->mDate); zipfileWrite32(a, pCds->crc32); zipfileWrite32(a, pCds->szCompressed); zipfileWrite32(a, pCds->szUncompressed); zipfileWrite16(a, (u16)pCds->nFile); zipfileWrite16(a, pCds->nExtra); assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); /* Add the file name */ memcpy(a, pCds->zFile, (int)pCds->nFile); a += (int)pCds->nFile; /* The "extra" data */ zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); zipfileWrite16(a, 5); *a++ = 0x01; zipfileWrite32(a, pEntry->mUnixTime); return a-aBuf; } static int zipfileAppendEntry( ZipfileTab *pTab, ZipfileEntry *pEntry, const u8 *pData, int nData ){ u8 *aBuf = pTab->aBuffer; int nBuf; int rc; nBuf = zipfileSerializeLFH(pEntry, aBuf); rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ pEntry->iDataOff = pTab->szCurrent; rc = zipfileAppendData(pTab, pData, nData); } return rc; } static int zipfileGetMode( sqlite3_value *pVal, int bIsDir, /* If true, default to directory */ u32 *pMode, /* OUT: Mode value */ char **pzErr /* OUT: Error message */ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ const char zTemplate[11] = "-rwxrwxrwx"; int i; if( strlen(z)!=10 ) goto parse_error; switch( z[0] ){ case '-': mode |= S_IFREG; break; case 'd': mode |= S_IFDIR; break; case 'l': mode |= S_IFLNK; break; default: goto parse_error; } for(i=1; i<10; i++){ if( z[i]==zTemplate[i] ) mode |= 1 << (9-i); else if( z[i]!='-' ) goto parse_error; } } if( ((mode & S_IFDIR)==0)==bIsDir ){ /* The "mode" attribute is a directory, but data has been specified. ** Or vice-versa - no data but "mode" is a file or symlink. */ *pzErr = sqlite3_mprintf("zipfile: mode does not match data"); return SQLITE_CONSTRAINT; } *pMode = mode; return SQLITE_OK; parse_error: *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); return SQLITE_ERROR; } /* ** Both (const char*) arguments point to nul-terminated strings. Argument ** nB is the value of strlen(zB). This function returns 0 if the strings are ** identical, ignoring any trailing '/' character in either path. */ static int zipfileComparePath(const char *zA, const char *zB, int nB){ int nA = (int)strlen(zA); if( nA>0 && zA[nA-1]=='/' ) nA--; if( nB>0 && zB[nB-1]=='/' ) nB--; if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; return 1; } static int zipfileBegin(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; assert( pTab->pWriteFd==0 ); if( pTab->zFile==0 || pTab->zFile[0]==0 ){ pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); return SQLITE_ERROR; } /* Open a write fd on the file. Also load the entire central directory ** structure into memory. During the transaction any new file data is ** appended to the archive file, but the central directory is accumulated ** in main-memory until the transaction is committed. */ pTab->pWriteFd = fopen(pTab->zFile, "ab+"); if( pTab->pWriteFd==0 ){ pTab->base.zErrMsg = sqlite3_mprintf( "zipfile: failed to open file %s for writing", pTab->zFile ); rc = SQLITE_ERROR; }else{ fseek(pTab->pWriteFd, 0, SEEK_END); pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); rc = zipfileLoadDirectory(pTab, 0, 0); } if( rc!=SQLITE_OK ){ zipfileCleanupTransaction(pTab); } return rc; } /* ** Return the current time as a 32-bit timestamp in UNIX epoch format (like ** time(2)). */ static u32 zipfileTime(void){ sqlite3_vfs *pVfs = sqlite3_vfs_find(0); u32 ret; if( pVfs==0 ) return 0; if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ i64 ms; pVfs->xCurrentTimeInt64(pVfs, &ms); ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); }else{ double day; pVfs->xCurrentTime(pVfs, &day); ret = (u32)((day - 2440587.5) * 86400); } return ret; } /* ** Return a 32-bit timestamp in UNIX epoch format. ** ** If the value passed as the only argument is either NULL or an SQL NULL, ** return the current time. Otherwise, return the value stored in (*pVal) ** cast to a 32-bit unsigned integer. */ static u32 zipfileGetTime(sqlite3_value *pVal){ if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ return zipfileTime(); } return (u32)sqlite3_value_int64(pVal); } /* ** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry ** linked list. Remove it from the list and free the object. */ static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ if( pOld ){ if( pTab->pFirstEntry==pOld ){ pTab->pFirstEntry = pOld->pNext; if( pTab->pLastEntry==pOld ) pTab->pLastEntry = 0; }else{ ZipfileEntry *p; for(p=pTab->pFirstEntry; p; p=p->pNext){ if( p->pNext==pOld ){ p->pNext = pOld->pNext; if( pTab->pLastEntry==pOld ) pTab->pLastEntry = p; break; } } } zipfileEntryFree(pOld); } } /* ** xUpdate method. */ static int zipfileUpdate( sqlite3_vtab *pVtab, int nVal, sqlite3_value **apVal, sqlite_int64 *pRowid ){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; /* Return Code */ ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ u32 mode = 0; /* Mode for new entry */ u32 mTime = 0; /* Modification time for new entry */ i64 sz = 0; /* Uncompressed size */ const char *zPath = 0; /* Path for new entry */ int nPath = 0; /* strlen(zPath) */ const u8 *pData = 0; /* Pointer to buffer containing content */ int nData = 0; /* Size of pData buffer in bytes */ int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ char *zFree = 0; /* Also free this */ ZipfileEntry *pOld = 0; ZipfileEntry *pOld2 = 0; int bUpdate = 0; /* True for an update that modifies "name" */ int bIsDir = 0; u32 iCrc32 = 0; (void)pRowid; if( pTab->pWriteFd==0 ){ rc = zipfileBegin(pVtab); if( rc!=SQLITE_OK ) return rc; } /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); int nDelete = (int)strlen(zDelete); if( nVal>1 ){ const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]); if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){ bUpdate = 1; } } for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ break; } assert( pOld->pNext ); } } if( nVal>1 ){ /* Check that "sz" and "rawdata" are both NULL: */ if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){ zipfileTableErr(pTab, "sz must be NULL"); rc = SQLITE_CONSTRAINT; } if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){ zipfileTableErr(pTab, "rawdata must be NULL"); rc = SQLITE_CONSTRAINT; } if( rc==SQLITE_OK ){ if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ /* data=NULL. A directory */ bIsDir = 1; }else{ /* Value specified for "data", and possibly "method". This must be ** a regular file or a symlink. */ const u8 *aIn = sqlite3_value_blob(apVal[7]); int nIn = sqlite3_value_bytes(apVal[7]); int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; iMethod = sqlite3_value_int(apVal[8]); sz = nIn; pData = aIn; nData = nIn; if( iMethod!=0 && iMethod!=8 ){ zipfileTableErr(pTab, "unknown compression method: %d", iMethod); rc = SQLITE_CONSTRAINT; }else{ if( bAuto || iMethod ){ int nCmp; rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ if( iMethod || nCmp<nIn ){ iMethod = 8; pData = pFree; nData = nCmp; } } } iCrc32 = crc32(0, aIn, nIn); } } } if( rc==SQLITE_OK ){ rc = zipfileGetMode(apVal[3], bIsDir, &mode, &pTab->base.zErrMsg); } if( rc==SQLITE_OK ){ zPath = (const char*)sqlite3_value_text(apVal[2]); if( zPath==0 ) zPath = ""; nPath = (int)strlen(zPath); mTime = zipfileGetTime(apVal[4]); } if( rc==SQLITE_OK && bIsDir ){ /* For a directory, check that the last character in the path is a ** '/'. This appears to be required for compatibility with info-zip ** (the unzip command on unix). It does not create directories ** otherwise. */ if( nPath<=0 || zPath[nPath-1]!='/' ){ zFree = sqlite3_mprintf("%s/", zPath); zPath = (const char*)zFree; if( zFree==0 ){ rc = SQLITE_NOMEM; nPath = 0; }else{ nPath = (int)strlen(zPath); } } } /* Check that we're not inserting a duplicate entry -OR- updating an ** entry with a path, thereby making it into a duplicate. */ if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){ ZipfileEntry *p; for(p=pTab->pFirstEntry; p; p=p->pNext){ if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ switch( sqlite3_vtab_on_conflict(pTab->db) ){ case SQLITE_IGNORE: { goto zipfile_update_done; } case SQLITE_REPLACE: { pOld2 = p; break; } default: { zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath); rc = SQLITE_CONSTRAINT; break; } } break; } } } if( rc==SQLITE_OK ){ /* Create the new CDS record. */ pNew = zipfileNewEntry(zPath); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; pNew->cds.iCompression = (u16)iMethod; zipfileMtimeToDos(&pNew->cds, mTime); pNew->cds.crc32 = iCrc32; pNew->cds.szCompressed = nData; pNew->cds.szUncompressed = (u32)sz; pNew->cds.iExternalAttr = (mode<<16); pNew->cds.iOffset = (u32)pTab->szCurrent; pNew->cds.nFile = (u16)nPath; pNew->mUnixTime = (u32)mTime; rc = zipfileAppendEntry(pTab, pNew, pData, nData); zipfileAddEntry(pTab, pOld, pNew); } } } if( rc==SQLITE_OK && (pOld || pOld2) ){ ZipfileCsr *pCsr; for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){ pCsr->pCurrent = pCsr->pCurrent->pNext; pCsr->bNoop = 1; } } zipfileRemoveEntryFromList(pTab, pOld); zipfileRemoveEntryFromList(pTab, pOld2); } zipfile_update_done: sqlite3_free(pFree); sqlite3_free(zFree); return rc; } static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ u8 *a = aBuf; zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); zipfileWrite16(a, p->iDisk); zipfileWrite16(a, p->iFirstDisk); zipfileWrite16(a, p->nEntry); zipfileWrite16(a, p->nEntryTotal); zipfileWrite32(a, p->nSize); zipfileWrite32(a, p->iOffset); zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ return a-aBuf; } static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } /* ** Serialize the CDS structure into buffer aBuf[]. Return the number ** of bytes written. */ static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ u8 *a = aBuf; ZipfileCDS *pCDS = &pEntry->cds; if( pEntry->aExtra==0 ){ pCDS->nExtra = 9; } zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); zipfileWrite16(a, pCDS->iVersionMadeBy); zipfileWrite16(a, pCDS->iVersionExtract); zipfileWrite16(a, pCDS->flags); zipfileWrite16(a, pCDS->iCompression); zipfileWrite16(a, pCDS->mTime); zipfileWrite16(a, pCDS->mDate); zipfileWrite32(a, pCDS->crc32); zipfileWrite32(a, pCDS->szCompressed); zipfileWrite32(a, pCDS->szUncompressed); assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); zipfileWrite16(a, pCDS->nFile); zipfileWrite16(a, pCDS->nExtra); zipfileWrite16(a, pCDS->nComment); zipfileWrite16(a, pCDS->iDiskStart); zipfileWrite16(a, pCDS->iInternalAttr); zipfileWrite32(a, pCDS->iExternalAttr); zipfileWrite32(a, pCDS->iOffset); memcpy(a, pCDS->zFile, pCDS->nFile); a += pCDS->nFile; if( pEntry->aExtra ){ int n = (int)pCDS->nExtra + (int)pCDS->nComment; memcpy(a, pEntry->aExtra, n); a += n; }else{ assert( pCDS->nExtra==9 ); zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); zipfileWrite16(a, 5); *a++ = 0x01; zipfileWrite32(a, pEntry->mUnixTime); } return a-aBuf; } static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; if( pTab->pWriteFd ){ i64 iOffset = pTab->szCurrent; ZipfileEntry *p; ZipfileEOCD eocd; int nEntry = 0; /* Write out all entries */ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ int n = zipfileSerializeCDS(p, pTab->aBuffer); rc = zipfileAppendData(pTab, pTab->aBuffer, n); nEntry++; } /* Write out the EOCD record */ eocd.iDisk = 0; eocd.iFirstDisk = 0; eocd.nEntry = (u16)nEntry; eocd.nEntryTotal = (u16)nEntry; eocd.nSize = (u32)(pTab->szCurrent - iOffset); eocd.iOffset = (u32)iOffset; rc = zipfileAppendEOCD(pTab, &eocd); zipfileCleanupTransaction(pTab); } return rc; } static int zipfileRollback(sqlite3_vtab *pVtab){ return zipfileCommit(pVtab); } static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){ ZipfileCsr *pCsr; for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ if( iId==pCsr->iId ) break; } return pCsr; } static void zipfileFunctionCds( sqlite3_context *context, int argc, sqlite3_value **argv ){ ZipfileCsr *pCsr; ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); assert( argc>0 ); pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); if( pCsr ){ ZipfileCDS *p = &pCsr->pCurrent->cds; char *zRes = sqlite3_mprintf("{" "\"version-made-by\" : %u, " "\"version-to-extract\" : %u, " "\"flags\" : %u, " "\"compression\" : %u, " "\"time\" : %u, " "\"date\" : %u, " "\"crc32\" : %u, " "\"compressed-size\" : %u, " "\"uncompressed-size\" : %u, " "\"file-name-length\" : %u, " "\"extra-field-length\" : %u, " "\"file-comment-length\" : %u, " "\"disk-number-start\" : %u, " "\"internal-attr\" : %u, " "\"external-attr\" : %u, " "\"offset\" : %u }", (u32)p->iVersionMadeBy, (u32)p->iVersionExtract, (u32)p->flags, (u32)p->iCompression, (u32)p->mTime, (u32)p->mDate, (u32)p->crc32, (u32)p->szCompressed, (u32)p->szUncompressed, (u32)p->nFile, (u32)p->nExtra, (u32)p->nComment, (u32)p->iDiskStart, (u32)p->iInternalAttr, (u32)p->iExternalAttr, (u32)p->iOffset ); if( zRes==0 ){ sqlite3_result_error_nomem(context); }else{ sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT); sqlite3_free(zRes); } } } /* ** xFindFunction method. */ static int zipfileFindFunction( sqlite3_vtab *pVtab, /* Virtual table handle */ int nArg, /* Number of SQL function arguments */ const char *zName, /* Name of SQL function */ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ (void)nArg; if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ *pxFunc = zipfileFunctionCds; *ppArg = (void*)pVtab; return 1; } return 0; } typedef struct ZipfileBuffer ZipfileBuffer; struct ZipfileBuffer { u8 *a; /* Pointer to buffer */ int n; /* Size of buffer in bytes */ int nAlloc; /* Byte allocated at a[] */ }; typedef struct ZipfileCtx ZipfileCtx; struct ZipfileCtx { int nEntry; ZipfileBuffer body; ZipfileBuffer cds; }; static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ if( pBuf->n+nByte>pBuf->nAlloc ){ u8 *aNew; sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; int nReq = pBuf->n + nByte; while( nNew<nReq ) nNew = nNew*2; aNew = sqlite3_realloc64(pBuf->a, nNew); if( aNew==0 ) return SQLITE_NOMEM; pBuf->a = aNew; pBuf->nAlloc = (int)nNew; } return SQLITE_OK; } /* ** xStep() callback for the zipfile() aggregate. This can be called in ** any of the following ways: ** ** SELECT zipfile(name,data) ... ** SELECT zipfile(name,mode,mtime,data) ... ** SELECT zipfile(name,mode,mtime,data,method) ... */ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ ZipfileCtx *p; /* Aggregate function context */ ZipfileEntry e; /* New entry to add to zip archive */ sqlite3_value *pName = 0; sqlite3_value *pMode = 0; sqlite3_value *pMtime = 0; sqlite3_value *pData = 0; sqlite3_value *pMethod = 0; int bIsDir = 0; u32 mode; int rc = SQLITE_OK; char *zErr = 0; int iMethod = -1; /* Compression method to use (0 or 8) */ const u8 *aData = 0; /* Possibly compressed data for new entry */ int nData = 0; /* Size of aData[] in bytes */ int szUncompressed = 0; /* Size of data before compression */ u8 *aFree = 0; /* Free this before returning */ u32 iCrc32 = 0; /* crc32 of uncompressed data */ char *zName = 0; /* Path (name) of new entry */ int nName = 0; /* Size of zName in bytes */ char *zFree = 0; /* Free this before returning */ int nByte; memset(&e, 0, sizeof(e)); p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); if( p==0 ) return; /* Martial the arguments into stack variables */ if( nVal!=2 && nVal!=4 && nVal!=5 ){ zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); rc = SQLITE_ERROR; goto zipfile_step_out; } pName = apVal[0]; if( nVal==2 ){ pData = apVal[1]; }else{ pMode = apVal[1]; pMtime = apVal[2]; pData = apVal[3]; if( nVal==5 ){ pMethod = apVal[4]; } } /* Check that the 'name' parameter looks ok. */ zName = (char*)sqlite3_value_text(pName); nName = sqlite3_value_bytes(pName); if( zName==0 ){ zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); rc = SQLITE_ERROR; goto zipfile_step_out; } /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use ** deflate compression) or NULL (choose automatically). */ if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ iMethod = (int)sqlite3_value_int64(pMethod); if( iMethod!=0 && iMethod!=8 ){ zErr = sqlite3_mprintf("illegal method value: %d", iMethod); rc = SQLITE_ERROR; goto zipfile_step_out; } } /* Now inspect the data. If this is NULL, then the new entry must be a ** directory. Otherwise, figure out whether or not the data should ** be deflated or simply stored in the zip archive. */ if( sqlite3_value_type(pData)==SQLITE_NULL ){ bIsDir = 1; iMethod = 0; }else{ aData = sqlite3_value_blob(pData); szUncompressed = nData = sqlite3_value_bytes(pData); iCrc32 = crc32(0, aData, nData); if( iMethod<0 || iMethod==8 ){ int nOut = 0; rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); if( rc!=SQLITE_OK ){ goto zipfile_step_out; } if( iMethod==8 || nOut<nData ){ aData = aFree; nData = nOut; iMethod = 8; }else{ iMethod = 0; } } } /* Decode the "mode" argument. */ rc = zipfileGetMode(pMode, bIsDir, &mode, &zErr); if( rc ) goto zipfile_step_out; /* Decode the "mtime" argument. */ e.mUnixTime = zipfileGetTime(pMtime); /* If this is a directory entry, ensure that there is exactly one '/' ** at the end of the path. Or, if this is not a directory and the path ** ends in '/' it is an error. */ if( bIsDir==0 ){ if( nName>0 && zName[nName-1]=='/' ){ zErr = sqlite3_mprintf("non-directory name must not end with /"); rc = SQLITE_ERROR; goto zipfile_step_out; } }else{ if( nName==0 || zName[nName-1]!='/' ){ zName = zFree = sqlite3_mprintf("%s/", zName); if( zName==0 ){ rc = SQLITE_NOMEM; goto zipfile_step_out; } nName = (int)strlen(zName); }else{ while( nName>1 && zName[nName-2]=='/' ) nName--; } } /* Assemble the ZipfileEntry object for the new zip archive entry */ e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; e.cds.iCompression = (u16)iMethod; zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); e.cds.crc32 = iCrc32; e.cds.szCompressed = nData; e.cds.szUncompressed = szUncompressed; e.cds.iExternalAttr = (mode<<16); e.cds.iOffset = p->body.n; e.cds.nFile = (u16)nName; e.cds.zFile = zName; /* Append the LFH to the body of the new archive */ nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); /* Append the data to the body of the new archive */ if( nData>0 ){ if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; memcpy(&p->body.a[p->body.n], aData, nData); p->body.n += nData; } /* Append the CDS record to the directory of the new archive */ nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); /* Increment the count of entries in the archive */ p->nEntry++; zipfile_step_out: sqlite3_free(aFree); sqlite3_free(zFree); if( rc ){ if( zErr ){ sqlite3_result_error(pCtx, zErr, -1); }else{ sqlite3_result_error_code(pCtx, rc); } } sqlite3_free(zErr); } /* ** xFinalize() callback for zipfile aggregate function. */ static void zipfileFinal(sqlite3_context *pCtx){ ZipfileCtx *p; ZipfileEOCD eocd; sqlite3_int64 nZip; u8 *aZip; p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); if( p==0 ) return; if( p->nEntry>0 ){ memset(&eocd, 0, sizeof(eocd)); eocd.nEntry = (u16)p->nEntry; eocd.nEntryTotal = (u16)p->nEntry; eocd.nSize = p->cds.n; eocd.iOffset = p->body.n; nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; aZip = (u8*)sqlite3_malloc64(nZip); if( aZip==0 ){ sqlite3_result_error_nomem(pCtx); }else{ memcpy(aZip, p->body.a, p->body.n); memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); sqlite3_result_blob(pCtx, aZip, (int)nZip, zipfileFree); } } sqlite3_free(p->body.a); sqlite3_free(p->cds.a); } /* ** Register the "zipfile" virtual table. */ static int zipfileRegister(sqlite3 *db){ static sqlite3_module zipfileModule = { 1, /* iVersion */ zipfileConnect, /* xCreate */ zipfileConnect, /* xConnect */ zipfileBestIndex, /* xBestIndex */ zipfileDisconnect, /* xDisconnect */ zipfileDisconnect, /* xDestroy */ zipfileOpen, /* xOpen - open a cursor */ zipfileClose, /* xClose - close a cursor */ zipfileFilter, /* xFilter - configure scan constraints */ zipfileNext, /* xNext - advance a cursor */ zipfileEof, /* xEof - check for end of scan */ zipfileColumn, /* xColumn - read data */ 0, /* xRowid - read data */ zipfileUpdate, /* xUpdate */ zipfileBegin, /* xBegin */ 0, /* xSync */ zipfileCommit, /* xCommit */ zipfileRollback, /* xRollback */ zipfileFindFunction, /* xFindMethod */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollback */ 0, /* xShadowName */ 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, zipfileStep, zipfileFinal ); } assert( sizeof(i64)==8 ); assert( sizeof(u32)==4 ); assert( sizeof(u16)==2 ); assert( sizeof(u8)==1 ); return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ #define zipfileRegister … #endif #ifdef _WIN32 #endif int sqlite3_zipfile_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ return zipfileRegister(db); } /************************* End ../ext/misc/zipfile.c ********************/ /************************* Begin ../ext/misc/sqlar.c ******************/ /* ** 2017-12-17 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** Utility functions sqlar_compress() and sqlar_uncompress(). Useful ** for working with sqlar archives and used by the shell tool's built-in ** sqlar support. */ /* #include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #include <zlib.h> #include <assert.h> /* ** Implementation of the "sqlar_compress(X)" SQL function. ** ** If the type of X is SQLITE_BLOB, and compressing that blob using ** zlib utility function compress() yields a smaller blob, return the ** compressed blob. Otherwise, return a copy of X. ** ** SQLar uses the "zlib format" for compressed content. The zlib format ** contains a two-byte identification header and a four-byte checksum at ** the end. This is different from ZIP which uses the raw deflate format. ** ** Future enhancements to SQLar might add support for new compression formats. ** If so, those new formats will be identified by alternative headers in the ** compressed data. */ static void sqlarCompressFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ assert( argc==1 ); if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ const Bytef *pData = sqlite3_value_blob(argv[0]); uLong nData = sqlite3_value_bytes(argv[0]); uLongf nOut = compressBound(nData); Bytef *pOut; pOut = (Bytef*)sqlite3_malloc(nOut); if( pOut==0 ){ sqlite3_result_error_nomem(context); return; }else{ if( Z_OK!=compress(pOut, &nOut, pData, nData) ){ sqlite3_result_error(context, "error in compress()", -1); }else if( nOut<nData ){ sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT); }else{ sqlite3_result_value(context, argv[0]); } sqlite3_free(pOut); } }else{ sqlite3_result_value(context, argv[0]); } } /* ** Implementation of the "sqlar_uncompress(X,SZ)" SQL function ** ** Parameter SZ is interpreted as an integer. If it is less than or ** equal to zero, then this function returns a copy of X. Or, if ** SZ is equal to the size of X when interpreted as a blob, also ** return a copy of X. Otherwise, decompress blob X using zlib ** utility function uncompress() and return the results (another ** blob). */ static void sqlarUncompressFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ uLong nData; sqlite3_int64 sz; assert( argc==2 ); sz = sqlite3_value_int(argv[1]); if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); Bytef *pOut = sqlite3_malloc(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ sqlite3_result_error(context, "error in uncompress()", -1); }else{ sqlite3_result_blob(context, pOut, szf, SQLITE_TRANSIENT); } sqlite3_free(pOut); } } #ifdef _WIN32 #endif int sqlite3_sqlar_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, sqlarCompressFunc, 0, 0); if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, sqlarUncompressFunc, 0, 0); } return rc; } /************************* End ../ext/misc/sqlar.c ********************/ #endif /************************* Begin ../ext/expert/sqlite3expert.h ******************/ /* ** 2017 April 07 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* */ #if !defined(SQLITEEXPERT_H) #define SQLITEEXPERT_H … /* #include "sqlite3.h" */ sqlite3expert; /* ** Create a new sqlite3expert object. ** ** If successful, a pointer to the new object is returned and (*pzErr) set ** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to ** an English-language error message. In this case it is the responsibility ** of the caller to eventually free the error message buffer using ** sqlite3_free(). */ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr); /* ** Configure an sqlite3expert object. ** ** EXPERT_CONFIG_SAMPLE: ** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for ** each candidate index. This involves scanning and sorting the entire ** contents of each user database table once for each candidate index ** associated with the table. For large databases, this can be ** prohibitively slow. This option allows the sqlite3expert object to ** be configured so that sqlite_stat1 data is instead generated based on a ** subset of each table, or so that no sqlite_stat1 data is used at all. ** ** A single integer argument is passed to this option. If the value is less ** than or equal to zero, then no sqlite_stat1 data is generated or used by ** the analysis - indexes are recommended based on the database schema only. ** Or, if the value is 100 or greater, complete sqlite_stat1 data is ** generated for each candidate index (this is the default). Finally, if the ** value falls between 0 and 100, then it represents the percentage of user ** table rows that should be considered when generating sqlite_stat1 data. ** ** Examples: ** ** // Do not generate any sqlite_stat1 data ** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0); ** ** // Generate sqlite_stat1 data based on 10% of the rows in each table. ** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10); */ int sqlite3_expert_config(sqlite3expert *p, int op, ...); #define EXPERT_CONFIG_SAMPLE … /* ** Specify zero or more SQL statements to be included in the analysis. ** ** Buffer zSql must contain zero or more complete SQL statements. This ** function parses all statements contained in the buffer and adds them ** to the internal list of statements to analyze. If successful, SQLITE_OK ** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example ** due to a error in the SQL - an SQLite error code is returned and (*pzErr) ** may be set to point to an English language error message. In this case ** the caller is responsible for eventually freeing the error message buffer ** using sqlite3_free(). ** ** If an error does occur while processing one of the statements in the ** buffer passed as the second argument, none of the statements in the ** buffer are added to the analysis. ** ** This function must be called before sqlite3_expert_analyze(). If a call ** to this function is made on an sqlite3expert object that has already ** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned ** immediately and no statements are added to the analysis. */ int sqlite3_expert_sql( sqlite3expert *p, /* From a successful sqlite3_expert_new() */ const char *zSql, /* SQL statement(s) to add */ char **pzErr /* OUT: Error message (if any) */ ); /* ** This function is called after the sqlite3expert object has been configured ** with all SQL statements using sqlite3_expert_sql() to actually perform ** the analysis. Once this function has been called, it is not possible to ** add further SQL statements to the analysis. ** ** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if ** an error occurs, an SQLite error code is returned and (*pzErr) set to ** point to a buffer containing an English language error message. In this ** case it is the responsibility of the caller to eventually free the buffer ** using sqlite3_free(). ** ** If an error does occur within this function, the sqlite3expert object ** is no longer useful for any purpose. At that point it is no longer ** possible to add further SQL statements to the object or to re-attempt ** the analysis. The sqlite3expert object must still be freed using a call ** sqlite3_expert_destroy(). */ int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr); /* ** Return the total number of statements loaded using sqlite3_expert_sql(). ** The total number of SQL statements may be different from the total number ** to calls to sqlite3_expert_sql(). */ int sqlite3_expert_count(sqlite3expert*); /* ** Return a component of the report. ** ** This function is called after sqlite3_expert_analyze() to extract the ** results of the analysis. Each call to this function returns either a ** NULL pointer or a pointer to a buffer containing a nul-terminated string. ** The value passed as the third argument must be one of the EXPERT_REPORT_* ** #define constants defined below. ** ** For some EXPERT_REPORT_* parameters, the buffer returned contains ** information relating to a specific SQL statement. In these cases that ** SQL statement is identified by the value passed as the second argument. ** SQL statements are numbered from 0 in the order in which they are parsed. ** If an out-of-range value (less than zero or equal to or greater than the ** value returned by sqlite3_expert_count()) is passed as the second argument ** along with such an EXPERT_REPORT_* parameter, NULL is always returned. ** ** EXPERT_REPORT_SQL: ** Return the text of SQL statement iStmt. ** ** EXPERT_REPORT_INDEXES: ** Return a buffer containing the CREATE INDEX statements for all recommended ** indexes for statement iStmt. If there are no new recommeded indexes, NULL ** is returned. ** ** EXPERT_REPORT_PLAN: ** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query ** iStmt after the proposed indexes have been added to the database schema. ** ** EXPERT_REPORT_CANDIDATES: ** Return a pointer to a buffer containing the CREATE INDEX statements ** for all indexes that were tested (for all SQL statements). The iStmt ** parameter is ignored for EXPERT_REPORT_CANDIDATES calls. */ const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport); /* ** Values for the third argument passed to sqlite3_expert_report(). */ #define EXPERT_REPORT_SQL … #define EXPERT_REPORT_INDEXES … #define EXPERT_REPORT_PLAN … #define EXPERT_REPORT_CANDIDATES … /* ** Free an (sqlite3expert*) handle and all associated resources. There ** should be one call to this function for each successful call to ** sqlite3-expert_new(). */ void sqlite3_expert_destroy(sqlite3expert*); #endif /* !defined(SQLITEEXPERT_H) */ /************************* End ../ext/expert/sqlite3expert.h ********************/ /************************* Begin ../ext/expert/sqlite3expert.c ******************/ /* ** 2017 April 09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* */ /* #include "sqlite3expert.h" */ #include <assert.h> #include <string.h> #include <stdio.h> #if !defined(SQLITE_AMALGAMATION) #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) #define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS … #endif #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) #define ALWAYS … #define NEVER … #elif !defined(NDEBUG) #define ALWAYS(X) … #define NEVER(X) … #else #define ALWAYS … #define NEVER … #endif #endif /* !defined(SQLITE_AMALGAMATION) */ #ifndef SQLITE_OMIT_VIRTUALTABLE /* typedef sqlite3_int64 i64; */ /* typedef sqlite3_uint64 u64; */ IdxColumn; IdxConstraint; IdxScan; IdxStatement; IdxTable; IdxWrite; #define STRLEN … /* ** A temp table name that we assume no user database will actually use. ** If this assumption proves incorrect triggers on the table with the ** conflicting name will be ignored. */ #define UNIQUE_TABLE_NAME … /* ** A single constraint. Equivalent to either "col = ?" or "col < ?" (or ** any other type of single-ended range constraint on a column). ** ** pLink: ** Used to temporarily link IdxConstraint objects into lists while ** creating candidate indexes. */ struct IdxConstraint { … }; /* ** A single scan of a single table. */ struct IdxScan { … }; /* ** Information regarding a single database table. Extracted from ** "PRAGMA table_info" by function idxGetTableInfo(). */ struct IdxColumn { … }; struct IdxTable { … }; /* ** An object of the following type is created for each unique table/write-op ** seen. The objects are stored in a singly-linked list beginning at ** sqlite3expert.pWrite. */ struct IdxWrite { … }; /* ** Each statement being analyzed is represented by an instance of this ** structure. */ struct IdxStatement { … }; /* ** A hash table for storing strings. With space for a payload string ** with each entry. Methods are: ** ** idxHashInit() ** idxHashClear() ** idxHashAdd() ** idxHashSearch() */ #define IDX_HASH_SIZE … IdxHashEntry; IdxHash; struct IdxHashEntry { … }; struct IdxHash { … }; /* ** sqlite3expert object. */ struct sqlite3expert { … }; /* ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. */ static void *idxMalloc(int *pRc, int nByte){ … } /* ** Initialize an IdxHash hash table. */ static void idxHashInit(IdxHash *pHash){ … } /* ** Reset an IdxHash hash table. */ static void idxHashClear(IdxHash *pHash){ … } /* ** Return the index of the hash bucket that the string specified by the ** arguments to this function belongs. */ static int idxHashString(const char *z, int n){ … } /* ** If zKey is already present in the hash table, return non-zero and do ** nothing. Otherwise, add an entry with key zKey and payload string zVal to ** the hash table passed as the second argument. */ static int idxHashAdd( int *pRc, IdxHash *pHash, const char *zKey, const char *zVal ){ … } /* ** If zKey/nKey is present in the hash table, return a pointer to the ** hash-entry object. */ static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){ … } /* ** If the hash table contains an entry with a key equal to the string ** passed as the final two arguments to this function, return a pointer ** to the payload string. Otherwise, if zKey/nKey is not present in the ** hash table, return NULL. */ static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){ … } /* ** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl ** variable to point to a copy of nul-terminated string zColl. */ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){ … } /* ** An error associated with database handle db has just occurred. Pass ** the error message to callback function xOut. */ static void idxDatabaseError( sqlite3 *db, /* Database handle */ char **pzErrmsg /* Write error here */ ){ … } /* ** Prepare an SQL statement. */ static int idxPrepareStmt( sqlite3 *db, /* Database handle to compile against */ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ const char *zSql /* SQL statement to compile */ ){ … } /* ** Prepare an SQL statement using the results of a printf() formatting. */ static int idxPrintfPrepareStmt( sqlite3 *db, /* Database handle to compile against */ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */ char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */ const char *zFmt, /* printf() format of SQL statement */ ... /* Trailing printf() arguments */ ){ … } /************************************************************************* ** Beginning of virtual table implementation. */ ExpertVtab; struct ExpertVtab { … }; ExpertCsr; struct ExpertCsr { … }; static char *expertDequote(const char *zIn){ … } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the r-tree virtual table. ** ** argv[0] -> module name ** argv[1] -> database name ** argv[2] -> table name ** argv[...] -> column names... */ static int expertConnect( sqlite3 *db, void *pAux, int argc, const char *const*argv, sqlite3_vtab **ppVtab, char **pzErr ){ … } static int expertDisconnect(sqlite3_vtab *pVtab){ … } static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){ … } static int expertUpdate( sqlite3_vtab *pVtab, int nData, sqlite3_value **azData, sqlite_int64 *pRowid ){ … } /* ** Virtual table module xOpen method. */ static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ … } /* ** Virtual table module xClose method. */ static int expertClose(sqlite3_vtab_cursor *cur){ … } /* ** Virtual table module xEof method. ** ** Return non-zero if the cursor does not currently point to a valid ** record (i.e if the scan has finished), or zero otherwise. */ static int expertEof(sqlite3_vtab_cursor *cur){ … } /* ** Virtual table module xNext method. */ static int expertNext(sqlite3_vtab_cursor *cur){ … } /* ** Virtual table module xRowid method. */ static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ … } /* ** Virtual table module xColumn method. */ static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ … } /* ** Virtual table module xFilter method. */ static int expertFilter( sqlite3_vtab_cursor *cur, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ … } static int idxRegisterVtab(sqlite3expert *p){ … } /* ** End of virtual table implementation. *************************************************************************/ /* ** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function ** is called, set it to the return value of sqlite3_finalize() before ** returning. Otherwise, discard the sqlite3_finalize() return value. */ static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){ … } /* ** Attempt to allocate an IdxTable structure corresponding to table zTab ** in the main database of connection db. If successful, set (*ppOut) to ** point to the new object and return SQLITE_OK. Otherwise, return an ** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be ** set to point to an error string. ** ** It is the responsibility of the caller to eventually free either the ** IdxTable object or error message using sqlite3_free(). */ static int idxGetTableInfo( sqlite3 *db, /* Database connection to read details from */ const char *zTab, /* Table name */ IdxTable **ppOut, /* OUT: New object (if successful) */ char **pzErrmsg /* OUT: Error message (if not) */ ){ … } /* ** This function is a no-op if *pRc is set to anything other than ** SQLITE_OK when it is called. ** ** If *pRc is initially set to SQLITE_OK, then the text specified by ** the printf() style arguments is appended to zIn and the result returned ** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on ** zIn before returning. */ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ … } /* ** Return true if zId must be quoted in order to use it as an SQL ** identifier, or false otherwise. */ static int idxIdentifierRequiresQuotes(const char *zId){ … } /* ** This function appends an index column definition suitable for constraint ** pCons to the string passed as zIn and returns the result. */ static char *idxAppendColDefn( int *pRc, /* IN/OUT: Error code */ char *zIn, /* Column defn accumulated so far */ IdxTable *pTab, /* Table index will be created on */ IdxConstraint *pCons ){ … } /* ** Search database dbm for an index compatible with the one idxCreateFromCons() ** would create from arguments pScan, pEq and pTail. If no error occurs and ** such an index is found, return non-zero. Or, if no such index is found, ** return zero. ** ** If an error occurs, set *pRc to an SQLite error code and return zero. */ static int idxFindCompatible( int *pRc, /* OUT: Error code */ sqlite3* dbm, /* Database to search */ IdxScan *pScan, /* Scan for table to search for index on */ IdxConstraint *pEq, /* List of == constraints */ IdxConstraint *pTail /* List of range constraints */ ){ … } /* Callback for sqlite3_exec() with query with leading count(*) column. * The first argument is expected to be an int*, referent to be incremented * if that leading column is not exactly '0'. */ static int countNonzeros(void* pCount, int nc, char* azResults[], char* azColumns[]){ … } static int idxCreateFromCons( sqlite3expert *p, IdxScan *pScan, IdxConstraint *pEq, IdxConstraint *pTail ){ … } /* ** Return true if list pList (linked by IdxConstraint.pLink) contains ** a constraint compatible with *p. Otherwise return false. */ static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){ … } static int idxCreateFromWhere( sqlite3expert *p, IdxScan *pScan, /* Create indexes for this scan */ IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */ ){ … } /* ** Create candidate indexes in database [dbm] based on the data in ** linked-list pScan. */ static int idxCreateCandidates(sqlite3expert *p){ … } /* ** Free all elements of the linked list starting at pConstraint. */ static void idxConstraintFree(IdxConstraint *pConstraint){ … } /* ** Free all elements of the linked list starting from pScan up until pLast ** (pLast is not freed). */ static void idxScanFree(IdxScan *pScan, IdxScan *pLast){ … } /* ** Free all elements of the linked list starting from pStatement up ** until pLast (pLast is not freed). */ static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){ … } /* ** Free the linked list of IdxTable objects starting at pTab. */ static void idxTableFree(IdxTable *pTab){ … } /* ** Free the linked list of IdxWrite objects starting at pTab. */ static void idxWriteFree(IdxWrite *pTab){ … } /* ** This function is called after candidate indexes have been created. It ** runs all the queries to see which indexes they prefer, and populates ** IdxStatement.zIdx and IdxStatement.zEQP with the results. */ static int idxFindIndexes( sqlite3expert *p, char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ … } static int idxAuthCallback( void *pCtx, int eOp, const char *z3, const char *z4, const char *zDb, const char *zTrigger ){ … } static int idxProcessOneTrigger( sqlite3expert *p, IdxWrite *pWrite, char **pzErr ){ … } static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ … } static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ … } struct IdxSampleCtx { … }; static void idxSampleFunc( sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ … } struct IdxRemCtx { … }; /* ** Implementation of scalar function rem(). */ static void idxRemFunc( sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ … } static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){ … } static int idxPopulateOneStat1( sqlite3expert *p, sqlite3_stmt *pIndexXInfo, sqlite3_stmt *pWriteStat, const char *zTab, const char *zIdx, char **pzErr ){ … } static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){ … } /* ** This function is called as part of sqlite3_expert_analyze(). Candidate ** indexes have already been created in database sqlite3expert.dbm, this ** function populates sqlite_stat1 table in the same database. ** ** The stat1 data is generated by querying the */ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ … } /* ** Define and possibly pretend to use a useless collation sequence. ** This pretense allows expert to accept SQL using custom collations. */ int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ … } /* And a callback to register above upon actual need */ void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ … } #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) /* ** dummy functions for no-op implementation of UDFs during expert's work */ void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ … } void dummyUDFvalue(sqlite3_context *up1){ … } /* ** Register UDFs from user database with another. */ int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ … } #endif /* ** Allocate a new sqlite3expert object. */ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ … } /* ** Configure an sqlite3expert object. */ int sqlite3_expert_config(sqlite3expert *p, int op, ...){ … } /* ** Add an SQL statement to the analysis. */ int sqlite3_expert_sql( sqlite3expert *p, /* From sqlite3_expert_new() */ const char *zSql, /* SQL statement to add */ char **pzErr /* OUT: Error message (if any) */ ){ … } int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){ … } /* ** Return the total number of statements that have been added to this ** sqlite3expert using sqlite3_expert_sql(). */ int sqlite3_expert_count(sqlite3expert *p){ … } /* ** Return a component of the report. */ const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){ … } /* ** Free an sqlite3expert object. */ void sqlite3_expert_destroy(sqlite3expert *p){ … } #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /************************* End ../ext/expert/sqlite3expert.c ********************/ /************************* Begin ../ext/intck/sqlite3intck.h ******************/ /* ** 2024-02-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* */ /* ** Incremental Integrity-Check Extension ** ------------------------------------- ** ** This module contains code to check whether or not an SQLite database ** is well-formed or corrupt. This is the same task as performed by SQLite's ** built-in "PRAGMA integrity_check" command. This module differs from ** "PRAGMA integrity_check" in that: ** ** + It is less thorough - this module does not detect certain types ** of corruption that are detected by the PRAGMA command. However, ** it does detect all kinds of corruption that are likely to cause ** errors in SQLite applications. ** ** + It is slower. Sometimes up to three times slower. ** ** + It allows integrity-check operations to be split into multiple ** transactions, so that the database does not need to be read-locked ** for the duration of the integrity-check. ** ** One way to use the API to run integrity-check on the "main" database ** of handle db is: ** ** int rc = SQLITE_OK; ** sqlite3_intck *p = 0; ** ** sqlite3_intck_open(db, "main", &p); ** while( SQLITE_OK==sqlite3_intck_step(p) ){ ** const char *zMsg = sqlite3_intck_message(p); ** if( zMsg ) printf("corruption: %s\n", zMsg); ** } ** rc = sqlite3_intck_error(p, &zErr); ** if( rc!=SQLITE_OK ){ ** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); ** } ** sqlite3_intck_close(p); ** ** Usually, the sqlite3_intck object opens a read transaction within the ** first call to sqlite3_intck_step() and holds it open until the ** integrity-check is complete. However, if sqlite3_intck_unlock() is ** called, the read transaction is ended and a new read transaction opened ** by the subsequent call to sqlite3_intck_step(). */ #ifndef _SQLITE_INTCK_H #define _SQLITE_INTCK_H /* #include "sqlite3.h" */ #ifdef __cplusplus extern "C" { #endif /* ** An ongoing incremental integrity-check operation is represented by an ** opaque pointer of the following type. */ sqlite3_intck; /* ** Open a new incremental integrity-check object. If successful, populate ** output variable (*ppOut) with the new object handle and return SQLITE_OK. ** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error ** code (e.g. SQLITE_NOMEM). ** ** The integrity-check will be conducted on database zDb (which must be "main", ** "temp", or the name of an attached database) of database handle db. Once ** this function has been called successfully, the caller should not use ** database handle db until the integrity-check object has been destroyed ** using sqlite3_intck_close(). */ int sqlite3_intck_open( sqlite3 *db, /* Database handle */ const char *zDb, /* Database name ("main", "temp" etc.) */ sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ ); /* ** Close and release all resources associated with a handle opened by an ** earlier call to sqlite3_intck_open(). The results of using an ** integrity-check handle after it has been passed to this function are ** undefined. */ void sqlite3_intck_close(sqlite3_intck *pCk); /* ** Do the next step of the integrity-check operation specified by the handle ** passed as the only argument. This function returns SQLITE_DONE if the ** integrity-check operation is finished, or an SQLite error code if ** an error occurs, or SQLITE_OK if no error occurs but the integrity-check ** is not finished. It is not considered an error if database corruption ** is encountered. ** ** Following a successful call to sqlite3_intck_step() (one that returns ** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if ** corruption was detected in the db. ** ** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is ** returned, then the integrity-check handle is placed in an error state. ** In this state all subsequent calls to sqlite3_intck_step() or ** sqlite3_intck_unlock() will immediately return the same error. The ** sqlite3_intck_error() method may be used to obtain an English language ** error message in this case. */ int sqlite3_intck_step(sqlite3_intck *pCk); /* ** If the previous call to sqlite3_intck_step() encountered corruption ** within the database, then this function returns a pointer to a buffer ** containing a nul-terminated string describing the corruption in ** English. If the previous call to sqlite3_intck_step() did not encounter ** corruption, or if there was no previous call, this function returns ** NULL. */ const char *sqlite3_intck_message(sqlite3_intck *pCk); /* ** Close any read-transaction opened by an earlier call to ** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will ** open a new transaction. Return SQLITE_OK if successful, or an SQLite error ** code otherwise. ** ** If an error occurs, then the integrity-check handle is placed in an error ** state. In this state all subsequent calls to sqlite3_intck_step() or ** sqlite3_intck_unlock() will immediately return the same error. The ** sqlite3_intck_error() method may be used to obtain an English language ** error message in this case. */ int sqlite3_intck_unlock(sqlite3_intck *pCk); /* ** If an error has occurred in an earlier call to sqlite3_intck_step() ** or sqlite3_intck_unlock(), then this method returns the associated ** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) ** may be set to point to a nul-terminated string containing an English ** language error message. Or, if no error message is available, to ** NULL. ** ** If no error has occurred within sqlite3_intck_step() or ** sqlite_intck_unlock() calls on the handle passed as the first argument, ** then SQLITE_OK is returned and (*pzErr) set to NULL. */ int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); /* ** This API is used for testing only. It returns the full-text of an SQL ** statement used to test object zObj, which may be a table or index. ** The returned buffer is valid until the next call to either this function ** or sqlite3_intck_close() on the same sqlite3_intck handle. */ const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); #ifdef __cplusplus } /* end of the 'extern "C"' block */ #endif #endif /* ifndef _SQLITE_INTCK_H */ /************************* End ../ext/intck/sqlite3intck.h ********************/ /************************* Begin ../ext/intck/sqlite3intck.c ******************/ /* ** 2024-02-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* */ /* #include "sqlite3intck.h" */ #include <string.h> #include <assert.h> #include <stdio.h> #include <stdlib.h> /* ** nKeyVal: ** The number of values that make up the 'key' for the current pCheck ** statement. ** ** rc: ** Error code returned by most recent sqlite3_intck_step() or ** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when ** the integrity-check operation is finished. ** ** zErr: ** If the object has entered the error state, this is the error message. ** Is freed using sqlite3_free() when the object is deleted. ** ** zTestSql: ** The value returned by the most recent call to sqlite3_intck_testsql(). ** Each call to testsql() frees the previous zTestSql value (using ** sqlite3_free()) and replaces it with the new value it will return. */ struct sqlite3_intck { … }; /* ** Some error has occurred while using database p->db. Save the error message ** and error code currently held by the database handle in p->rc and p->zErr. */ static void intckSaveErrmsg(sqlite3_intck *p){ … } /* ** If the handle passed as the first argument is already in the error state, ** then this function is a no-op (returns NULL immediately). Otherwise, if an ** error occurs within this function, it leaves an error in said handle. ** ** Otherwise, this function attempts to prepare SQL statement zSql and ** return the resulting statement handle to the user. */ static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ … } /* ** If the handle passed as the first argument is already in the error state, ** then this function is a no-op (returns NULL immediately). Otherwise, if an ** error occurs within this function, it leaves an error in said handle. ** ** Otherwise, this function treats argument zFmt as a printf() style format ** string. It formats it according to the trailing arguments and then ** attempts to prepare the results and return the resulting prepared ** statement. */ static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ … } /* ** Finalize SQL statement pStmt. If an error occurs and the handle passed ** as the first argument does not already contain an error, store the ** error in the handle. */ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ … } /* ** If there is already an error in handle p, return it. Otherwise, call ** sqlite3_step() on the statement handle and return that value. */ static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ … } /* ** Execute SQL statement zSql. There is no way to obtain any results ** returned by the statement. This function uses the sqlite3_intck error ** code convention. */ static void intckExec(sqlite3_intck *p, const char *zSql){ … } /* ** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error ** code convention. */ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ … } /* ** This is used by sqlite3_intck_unlock() to save the vector key value ** required to restart the current pCheck query as a nul-terminated string ** in p->zKey. */ static void intckSaveKey(sqlite3_intck *p){ … } /* ** Find the next database object (table or index) to check. If successful, ** set sqlite3_intck.zObj to point to a nul-terminated buffer containing ** the object's name before returning. */ static void intckFindObject(sqlite3_intck *p){ … } /* ** Return the size in bytes of the first token in nul-terminated buffer z. ** For the purposes of this call, a token is either: ** ** * a quoted SQL string, * * a contiguous series of ascii alphabet characters, or * * any other single byte. */ static int intckGetToken(const char *z){ … } /* ** Return true if argument c is an ascii whitespace character. */ static int intckIsSpace(char c){ … } /* ** Argument z points to the text of a CREATE INDEX statement. This function ** identifies the part of the text that contains either the index WHERE ** clause (if iCol<0) or the iCol'th column of the index. ** ** If (iCol<0), the identified fragment does not include the "WHERE" keyword, ** only the expression that follows it. If (iCol>=0) then the identified ** fragment does not include any trailing sort-order keywords - "ASC" or ** "DESC". ** ** If the CREATE INDEX statement does not contain the requested field or ** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to ** the identified fragment is returned and output parameter (*pnByte) set ** to its size in bytes. */ static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ … } /* ** User-defined SQL function wrapper for intckParseCreateIndex(): ** ** SELECT parse_create_index(<sql>, <icol>); */ static void intckParseCreateIndexFunc( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** Return true if sqlite3_intck.db has automatic indexes enabled, false ** otherwise. */ static int intckGetAutoIndex(sqlite3_intck *p){ … } /* ** Return true if zObj is an index, or false otherwise. */ static int intckIsIndex(sqlite3_intck *p, const char *zObj){ … } /* ** Return a pointer to a nul-terminated buffer containing the SQL statement ** used to check database object zObj (a table or index) for corruption. ** If parameter zPrev is not NULL, then it must be a string containing the ** vector key required to restart the check where it left off last time. ** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of ** columns in the vector key value for the specified object. ** ** This function uses the sqlite3_intck error code convention. */ static char *intckCheckObjectSql( sqlite3_intck *p, /* Integrity check object */ const char *zObj, /* Object (table or index) to scan */ const char *zPrev, /* Restart key vector, if any */ int *pnKeyVal /* OUT: Number of key-values for this scan */ ){ … } /* ** Open a new integrity-check object. */ int sqlite3_intck_open( sqlite3 *db, /* Database handle to operate on */ const char *zDbArg, /* "main", "temp" etc. */ sqlite3_intck **ppOut /* OUT: New integrity-check handle */ ){ … } /* ** Free the integrity-check object. */ void sqlite3_intck_close(sqlite3_intck *p){ … } /* ** Step the integrity-check object. */ int sqlite3_intck_step(sqlite3_intck *p){ … } /* ** Return a message describing the corruption encountered by the most recent ** call to sqlite3_intck_step(), or NULL if no corruption was encountered. */ const char *sqlite3_intck_message(sqlite3_intck *p){ … } /* ** Return the error code and message. */ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ … } /* ** Close any read transaction the integrity-check object is holding open ** on the database. */ int sqlite3_intck_unlock(sqlite3_intck *p){ … } /* ** Return the SQL statement used to check object zObj. Or, if zObj is ** NULL, the current SQL statement. */ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ … } /************************* End ../ext/intck/sqlite3intck.c ********************/ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER … #else #define SQLITE_SHELL_HAVE_RECOVER … #endif #if SQLITE_SHELL_HAVE_RECOVER /************************* Begin ../ext/recover/sqlite3recover.h ******************/ /* ** 2022-08-27 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This file contains the public interface to the "recover" extension - ** an SQLite extension designed to recover data from corrupted database ** files. */ /* ** OVERVIEW: ** ** To use the API to recover data from a corrupted database, an ** application: ** ** 1) Creates an sqlite3_recover handle by calling either ** sqlite3_recover_init() or sqlite3_recover_init_sql(). ** ** 2) Configures the new handle using one or more calls to ** sqlite3_recover_config(). ** ** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on ** the handle until it returns something other than SQLITE_OK. If it ** returns SQLITE_DONE, then the recovery operation completed without ** error. If it returns some other non-SQLITE_OK value, then an error ** has occurred. ** ** 4) Retrieves any error code and English language error message using the ** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, ** respectively. ** ** 5) Destroys the sqlite3_recover handle and frees all resources ** using sqlite3_recover_finish(). ** ** The application may abandon the recovery operation at any point ** before it is finished by passing the sqlite3_recover handle to ** sqlite3_recover_finish(). This is not an error, but the final state ** of the output database, or the results of running the partial script ** delivered to the SQL callback, are undefined. */ #ifndef _SQLITE_RECOVER_H #define _SQLITE_RECOVER_H /* #include "sqlite3.h" */ #ifdef __cplusplus extern "C" { #endif /* ** An instance of the sqlite3_recover object represents a recovery ** operation in progress. ** ** Constructors: ** ** sqlite3_recover_init() ** sqlite3_recover_init_sql() ** ** Destructor: ** ** sqlite3_recover_finish() ** ** Methods: ** ** sqlite3_recover_config() ** sqlite3_recover_errcode() ** sqlite3_recover_errmsg() ** sqlite3_recover_run() ** sqlite3_recover_step() */ typedef struct sqlite3_recover sqlite3_recover; /* ** These two APIs attempt to create and return a new sqlite3_recover object. ** In both cases the first two arguments identify the (possibly ** corrupt) database to recover data from. The first argument is an open ** database handle and the second the name of a database attached to that ** handle (i.e. "main", "temp" or the name of an attached database). ** ** If sqlite3_recover_init() is used to create the new sqlite3_recover ** handle, then data is recovered into a new database, identified by ** string parameter zUri. zUri may be an absolute or relative file path, ** or may be an SQLite URI. If the identified database file already exists, ** it is overwritten. ** ** If sqlite3_recover_init_sql() is invoked, then any recovered data will ** be returned to the user as a series of SQL statements. Executing these ** SQL statements results in the same database as would have been created ** had sqlite3_recover_init() been used. For each SQL statement in the ** output, the callback function passed as the third argument (xSql) is ** invoked once. The first parameter is a passed a copy of the fourth argument ** to this function (pCtx) as its first parameter, and a pointer to a ** nul-terminated buffer containing the SQL statement formated as UTF-8 as ** the second. If the xSql callback returns any value other than SQLITE_OK, ** then processing is immediately abandoned and the value returned used as ** the recover handle error code (see below). ** ** If an out-of-memory error occurs, NULL may be returned instead of ** a valid handle. In all other cases, it is the responsibility of the ** application to avoid resource leaks by ensuring that ** sqlite3_recover_finish() is called on all allocated handles. */ sqlite3_recover *sqlite3_recover_init( sqlite3* db, const char *zDb, const char *zUri ); sqlite3_recover *sqlite3_recover_init_sql( sqlite3* db, const char *zDb, int (*xSql)(void*, const char*), void *pCtx ); /* ** Configure an sqlite3_recover object that has just been created using ** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function ** may only be called before the first call to sqlite3_recover_step() ** or sqlite3_recover_run() on the object. ** ** The second argument passed to this function must be one of the ** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument ** depend on the specific SQLITE_RECOVER_* symbol in use. ** ** SQLITE_OK is returned if the configuration operation was successful, ** or an SQLite error code otherwise. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); /* ** SQLITE_RECOVER_LOST_AND_FOUND: ** The pArg argument points to a string buffer containing the name ** of a "lost-and-found" table in the output database, or NULL. If ** the argument is non-NULL and the database contains seemingly ** valid pages that cannot be associated with any table in the ** recovered part of the schema, data is extracted from these ** pages to add to the lost-and-found table. ** ** SQLITE_RECOVER_FREELIST_CORRUPT: ** The pArg value must actually be a pointer to a value of type ** int containing value 0 or 1 cast as a (void*). If this option is set ** (argument is 1) and a lost-and-found table has been configured using ** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is ** corrupt and an attempt is made to recover records from pages that ** appear to be linked into the freelist. Otherwise, pages on the freelist ** are ignored. Setting this option can recover more data from the ** database, but often ends up "recovering" deleted records. The default ** value is 0 (clear). ** ** SQLITE_RECOVER_ROWIDS: ** The pArg value must actually be a pointer to a value of type ** int containing value 0 or 1 cast as a (void*). If this option is set ** (argument is 1), then an attempt is made to recover rowid values ** that are not also INTEGER PRIMARY KEY values. If this option is ** clear, then new rowids are assigned to all recovered rows. The ** default value is 1 (set). ** ** SQLITE_RECOVER_SLOWINDEXES: ** The pArg value must actually be a pointer to a value of type ** int containing value 0 or 1 cast as a (void*). If this option is clear ** (argument is 0), then when creating an output database, the recover ** module creates and populates non-UNIQUE indexes right at the end of the ** recovery operation - after all recoverable data has been inserted ** into the new database. This is faster overall, but means that the ** final call to sqlite3_recover_step() for a recovery operation may ** be need to create a large number of indexes, which may be very slow. ** ** Or, if this option is set (argument is 1), then non-UNIQUE indexes ** are created in the output database before it is populated with ** recovered data. This is slower overall, but avoids the slow call ** to sqlite3_recover_step() at the end of the recovery operation. ** ** The default option value is 0. */ #define SQLITE_RECOVER_LOST_AND_FOUND … #define SQLITE_RECOVER_FREELIST_CORRUPT … #define SQLITE_RECOVER_ROWIDS … #define SQLITE_RECOVER_SLOWINDEXES … /* ** Perform a unit of work towards the recovery operation. This function ** must normally be called multiple times to complete database recovery. ** ** If no error occurs but the recovery operation is not completed, this ** function returns SQLITE_OK. If recovery has been completed successfully ** then SQLITE_DONE is returned. If an error has occurred, then an SQLite ** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not ** considered an error if some or all of the data cannot be recovered ** due to database corruption. ** ** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, ** all further such calls on the same recover handle are no-ops that return ** the same non-SQLITE_OK value. */ int sqlite3_recover_step(sqlite3_recover*); /* ** Run the recovery operation to completion. Return SQLITE_OK if successful, ** or an SQLite error code otherwise. Calling this function is the same ** as executing: ** ** while( SQLITE_OK==sqlite3_recover_step(p) ); ** return sqlite3_recover_errcode(p); */ int sqlite3_recover_run(sqlite3_recover*); /* ** If an error has been encountered during a prior call to ** sqlite3_recover_step(), then this function attempts to return a ** pointer to a buffer containing an English language explanation of ** the error. If no error message is available, or if an out-of memory ** error occurs while attempting to allocate a buffer in which to format ** the error message, NULL is returned. ** ** The returned buffer remains valid until the sqlite3_recover handle is ** destroyed using sqlite3_recover_finish(). */ const char *sqlite3_recover_errmsg(sqlite3_recover*); /* ** If this function is called on an sqlite3_recover handle after ** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. */ int sqlite3_recover_errcode(sqlite3_recover*); /* ** Clean up a recovery object created by a call to sqlite3_recover_init(). ** The results of using a recovery object with any API after it has been ** passed to this function are undefined. ** ** This function returns the same value as sqlite3_recover_errcode(). */ int sqlite3_recover_finish(sqlite3_recover*); #ifdef __cplusplus } /* end of the 'extern "C"' block */ #endif #endif /* ifndef _SQLITE_RECOVER_H */ /************************* End ../ext/recover/sqlite3recover.h ********************/ # ifndef SQLITE_HAVE_SQLITE3R /************************* Begin ../ext/recover/dbdata.c ******************/ /* ** 2019-04-17 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This file contains an implementation of two eponymous virtual tables, ** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the ** "sqlite_dbpage" eponymous virtual table be available. ** ** SQLITE_DBDATA: ** sqlite_dbdata is used to extract data directly from a database b-tree ** page and its associated overflow pages, bypassing the b-tree layer. ** The table schema is equivalent to: ** ** CREATE TABLE sqlite_dbdata( ** pgno INTEGER, ** cell INTEGER, ** field INTEGER, ** value ANY, ** schema TEXT HIDDEN ** ); ** ** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE ** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND ** "schema". ** ** Each page of the database is inspected. If it cannot be interpreted as ** a b-tree page, or if it is a b-tree page containing 0 entries, the ** sqlite_dbdata table contains no rows for that page. Otherwise, the ** table contains one row for each field in the record associated with ** each cell on the page. For intkey b-trees, the key value is stored in ** field -1. ** ** For example, for the database: ** ** CREATE TABLE t1(a, b); -- root page is page 2 ** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); ** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); ** ** the sqlite_dbdata table contains, as well as from entries related to ** page 1, content equivalent to: ** ** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES ** (2, 0, -1, 5 ), ** (2, 0, 0, 'v' ), ** (2, 0, 1, 'five'), ** (2, 1, -1, 10 ), ** (2, 1, 0, 'x' ), ** (2, 1, 1, 'ten' ); ** ** If database corruption is encountered, this module does not report an ** error. Instead, it attempts to extract as much data as possible and ** ignores the corruption. ** ** SQLITE_DBPTR: ** The sqlite_dbptr table has the following schema: ** ** CREATE TABLE sqlite_dbptr( ** pgno INTEGER, ** child INTEGER, ** schema TEXT HIDDEN ** ); ** ** It contains one entry for each b-tree pointer between a parent and ** child page in the database. */ #if !defined(SQLITEINT_H) /* #include "sqlite3.h" */ /* typedef unsigned char u8; */ /* typedef unsigned int u32; */ #endif #include <string.h> #include <assert.h> #ifndef SQLITE_OMIT_VIRTUALTABLE #define DBDATA_PADDING_BYTES … typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; typedef struct DbdataBuffer DbdataBuffer; /* ** Buffer type. */ struct DbdataBuffer { u8 *aBuf; sqlite3_int64 nBuf; }; /* Cursor object */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ sqlite3_stmt *pStmt; /* For fetching database pages */ int iPgno; /* Current page number */ u8 *aPage; /* Buffer containing page */ int nPage; /* Size of aPage[] in bytes */ int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ int bOnePage; /* True to stop after one page */ int szDb; sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ DbdataBuffer rec; sqlite3_int64 nRec; /* Size of pRec[] in bytes */ sqlite3_int64 nHdr; /* Size of header in bytes */ int iField; /* Current field number */ u8 *pHdrPtr; u8 *pPtr; u32 enc; /* Text encoding */ sqlite3_int64 iIntkey; /* Integer key value */ }; /* Table object */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ sqlite3_stmt *pStmt; /* For fetching database pages */ int bPtr; /* True for sqlite3_dbptr table */ }; /* Column and schema definitions for sqlite_dbdata */ #define DBDATA_COLUMN_PGNO … #define DBDATA_COLUMN_CELL … #define DBDATA_COLUMN_FIELD … #define DBDATA_COLUMN_VALUE … #define DBDATA_COLUMN_SCHEMA … #define DBDATA_SCHEMA … /* Column and schema definitions for sqlite_dbptr */ #define DBPTR_COLUMN_PGNO … #define DBPTR_COLUMN_CHILD … #define DBPTR_COLUMN_SCHEMA … #define DBPTR_SCHEMA … /* ** Ensure the buffer passed as the first argument is at least nMin bytes ** in size. If an error occurs while attempting to resize the buffer, ** SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ static int dbdataBufferSize(DbdataBuffer *pBuf, sqlite3_int64 nMin){ if( nMin>pBuf->nBuf ){ sqlite3_int64 nNew = nMin+16384; u8 *aNew = (u8*)sqlite3_realloc64(pBuf->aBuf, nNew); if( aNew==0 ) return SQLITE_NOMEM; pBuf->aBuf = aNew; pBuf->nBuf = nNew; } return SQLITE_OK; } /* ** Release the allocation managed by buffer pBuf. */ static void dbdataBufferFree(DbdataBuffer *pBuf){ sqlite3_free(pBuf->aBuf); memset(pBuf, 0, sizeof(*pBuf)); } /* ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual ** table. */ static int dbdataConnect( sqlite3 *db, void *pAux, int argc, const char *const*argv, sqlite3_vtab **ppVtab, char **pzErr ){ DbdataTable *pTab = 0; int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); (void)argc; (void)argv; (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ memset(pTab, 0, sizeof(DbdataTable)); pTab->db = db; pTab->bPtr = (pAux!=0); } } *ppVtab = (sqlite3_vtab*)pTab; return rc; } /* ** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. */ static int dbdataDisconnect(sqlite3_vtab *pVtab){ DbdataTable *pTab = (DbdataTable*)pVtab; if( pTab ){ sqlite3_finalize(pTab->pStmt); sqlite3_free(pVtab); } return SQLITE_OK; } /* ** This function interprets two types of constraints: ** ** schema=? ** pgno=? ** ** If neither are present, idxNum is set to 0. If schema=? is present, ** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit ** in idxNum is set. ** ** If both parameters are present, schema is in position 0 and pgno in ** position 1. */ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); for(i=0; i<pIdx->nConstraint; i++){ struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; iSchema = i; } if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ iPgno = i; } } } if( iSchema>=0 ){ pIdx->aConstraintUsage[iSchema].argvIndex = 1; pIdx->aConstraintUsage[iSchema].omit = 1; } if( iPgno>=0 ){ pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); pIdx->aConstraintUsage[iPgno].omit = 1; pIdx->estimatedCost = 100; pIdx->estimatedRows = 50; if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ int iCol = pIdx->aOrderBy[0].iColumn; if( pIdx->nOrderBy==1 ){ pIdx->orderByConsumed = (iCol==0 || iCol==1); }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); } } }else{ pIdx->estimatedCost = 100000000; pIdx->estimatedRows = 1000000000; } pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); return SQLITE_OK; } /* ** Open a new sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ DbdataCursor *pCsr; pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); if( pCsr==0 ){ return SQLITE_NOMEM; }else{ memset(pCsr, 0, sizeof(DbdataCursor)); pCsr->base.pVtab = pVTab; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; return SQLITE_OK; } /* ** Restore a cursor object to the state it was in when first allocated ** by dbdataOpen(). */ static void dbdataResetCursor(DbdataCursor *pCsr){ DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); if( pTab->pStmt==0 ){ pTab->pStmt = pCsr->pStmt; }else{ sqlite3_finalize(pCsr->pStmt); } pCsr->pStmt = 0; pCsr->iPgno = 1; pCsr->iCell = 0; pCsr->iField = 0; pCsr->bOnePage = 0; sqlite3_free(pCsr->aPage); dbdataBufferFree(&pCsr->rec); pCsr->aPage = 0; pCsr->nRec = 0; } /* ** Close an sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; dbdataResetCursor(pCsr); sqlite3_free(pCsr); return SQLITE_OK; } /* ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. */ static u32 get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } static u32 get_uint32(unsigned char *a){ return ((u32)a[0]<<24) | ((u32)a[1]<<16) | ((u32)a[2]<<8) | ((u32)a[3]); } /* ** Load page pgno from the database via the sqlite_dbpage virtual table. ** If successful, set (*ppPage) to point to a buffer containing the page ** data, (*pnPage) to the size of that buffer in bytes and return ** SQLITE_OK. In this case it is the responsibility of the caller to ** eventually free the buffer using sqlite3_free(). ** ** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and ** return an SQLite error code. */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ u32 pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ int rc2; int rc = SQLITE_OK; sqlite3_stmt *pStmt = pCsr->pStmt; *ppPage = 0; *pnPage = 0; if( pgno>0 ){ sqlite3_bind_int64(pStmt, 2, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ int nCopy = sqlite3_column_bytes(pStmt, 0); if( nCopy>0 ){ u8 *pPage; pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES); if( pPage==0 ){ rc = SQLITE_NOMEM; }else{ const u8 *pCopy = sqlite3_column_blob(pStmt, 0); memcpy(pPage, pCopy, nCopy); memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES); } *ppPage = pPage; *pnPage = nCopy; } } rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = rc2; } return rc; } /* ** Read a varint. Put the value in *pVal and return the number of bytes. */ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ u = (u<<7) + (z[i]&0x7f); if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } u = (u<<8) + (z[i]&0xff); *pVal = (sqlite3_int64)u; return 9; } /* ** Like dbdataGetVarint(), but set the output to 0 if it is less than 0 ** or greater than 0xFFFFFFFF. This can be used for all varints in an ** SQLite database except for key values in intkey tables. */ static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){ sqlite3_int64 val; int nRet = dbdataGetVarint(z, &val); if( val<0 || val>0xFFFFFFFF ) val = 0; *pVal = val; return nRet; } /* ** Return the number of bytes of space used by an SQLite value of type ** eType. */ static int dbdataValueBytes(int eType){ switch( eType ){ case 0: case 8: case 9: case 10: case 11: return 0; case 1: return 1; case 2: return 2; case 3: return 3; case 4: return 4; case 5: return 6; case 6: case 7: return 8; default: if( eType>0 ){ return ((eType-12) / 2); } return 0; } } /* ** Load a value of type eType from buffer pData and use it to set the ** result of context object pCtx. */ static void dbdataValue( sqlite3_context *pCtx, u32 enc, int eType, u8 *pData, sqlite3_int64 nData ){ if( eType>=0 ){ if( dbdataValueBytes(eType)<=nData ){ switch( eType ){ case 0: case 10: case 11: sqlite3_result_null(pCtx); break; case 8: sqlite3_result_int(pCtx, 0); break; case 9: sqlite3_result_int(pCtx, 1); break; case 1: case 2: case 3: case 4: case 5: case 6: case 7: { sqlite3_uint64 v = (signed char)pData[0]; pData++; switch( eType ){ case 7: case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; case 4: v = (v<<8) + pData[0]; pData++; case 3: v = (v<<8) + pData[0]; pData++; case 2: v = (v<<8) + pData[0]; pData++; } if( eType==7 ){ double r; memcpy(&r, &v, sizeof(r)); sqlite3_result_double(pCtx, r); }else{ sqlite3_result_int64(pCtx, (sqlite3_int64)v); } break; } default: { int n = ((eType-12) / 2); if( eType % 2 ){ switch( enc ){ #ifndef SQLITE_OMIT_UTF16 case SQLITE_UTF16BE: sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); break; case SQLITE_UTF16LE: sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); break; #endif default: sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); break; } }else{ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } } } }else{ if( eType==7 ){ sqlite3_result_double(pCtx, 0.0); }else if( eType<7 ){ sqlite3_result_int(pCtx, 0); }else if( eType%2 ){ sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); }else{ sqlite3_result_blob(pCtx, "", 0, SQLITE_STATIC); } } } } /* This macro is a copy of the MX_CELL() macro in the SQLite core. Given ** a page-size, it returns the maximum number of cells that may be present ** on the page. */ #define DBDATA_MX_CELL … /* Maximum number of fields that may appear in a single record. This is ** the "hard-limit", according to comments in sqliteLimit.h. */ #define DBDATA_MX_FIELD … /* ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; pCsr->iRowid++; while( 1 ){ int rc; int iOff = (pCsr->iPgno==1 ? 100 : 0); int bNextPage = 0; if( pCsr->aPage==0 ){ while( 1 ){ if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; if( pCsr->aPage && pCsr->nPage>=256 ) break; sqlite3_free(pCsr->aPage); pCsr->aPage = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); if( pCsr->nCell>DBDATA_MX_CELL(pCsr->nPage) ){ pCsr->nCell = DBDATA_MX_CELL(pCsr->nPage); } } if( pTab->bPtr ){ if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ pCsr->iCell = pCsr->nCell; } pCsr->iCell++; if( pCsr->iCell>=pCsr->nCell ){ sqlite3_free(pCsr->aPage); pCsr->aPage = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; }else{ return SQLITE_OK; } }else{ /* If there is no record loaded, load it now. */ assert( pCsr->rec.aBuf!=0 || pCsr->nRec==0 ); if( pCsr->nRec==0 ){ int bHasRowid = 0; int nPointer = 0; sqlite3_int64 nPayload = 0; sqlite3_int64 nHdr = 0; int iHdr; int U, X; int nLocal; switch( pCsr->aPage[iOff] ){ case 0x02: nPointer = 4; break; case 0x0a: break; case 0x0d: bHasRowid = 1; break; default: /* This is not a b-tree page with records on it. Continue. */ pCsr->iCell = pCsr->nCell; break; } if( pCsr->iCell>=pCsr->nCell ){ bNextPage = 1; }else{ int iCellPtr = iOff + 8 + nPointer + pCsr->iCell*2; if( iCellPtr>pCsr->nPage ){ bNextPage = 1; }else{ iOff = get_uint16(&pCsr->aPage[iCellPtr]); } /* For an interior node cell, skip past the child-page number */ iOff += nPointer; /* Load the "byte of payload including overflow" field */ if( bNextPage || iOff>pCsr->nPage || iOff<=iCellPtr ){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; if( nPayload==0 ) nPayload = 1; } /* If this is a leaf intkey cell, load the rowid */ if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){ iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); } /* Figure out how much data to read from the local page */ U = pCsr->nPage; if( bHasRowid ){ X = U-35; }else{ X = ((U-12)*64/255)-23; } if( nPayload<=X ){ nLocal = nPayload; }else{ int M, K; M = ((U-12)*32/255)-23; K = M+((nPayload-M)%(U-4)); if( K<=X ){ nLocal = K; }else{ nLocal = M; } } if( bNextPage || nLocal+iOff>pCsr->nPage ){ bNextPage = 1; }else{ /* Allocate space for payload. And a bit more to catch small buffer ** overruns caused by attempting to read a varint or similar from ** near the end of a corrupt record. */ rc = dbdataBufferSize(&pCsr->rec, nPayload+DBDATA_PADDING_BYTES); if( rc!=SQLITE_OK ) return rc; assert( nPayload!=0 ); /* Load the nLocal bytes of payload */ memcpy(pCsr->rec.aBuf, &pCsr->aPage[iOff], nLocal); iOff += nLocal; /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; int nCopy; rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage ); if( rc!=SQLITE_OK ) return rc; if( aOvfl==0 ) break; nCopy = U-4; if( nCopy>nRem ) nCopy = nRem; memcpy(&pCsr->rec.aBuf[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } nPayload -= nRem; } memset(&pCsr->rec.aBuf[nPayload], 0, DBDATA_PADDING_BYTES); pCsr->nRec = nPayload; iHdr = dbdataGetVarintU32(pCsr->rec.aBuf, &nHdr); if( nHdr>nPayload ) nHdr = 0; pCsr->nHdr = nHdr; pCsr->pHdrPtr = &pCsr->rec.aBuf[iHdr]; pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nHdr]; pCsr->iField = (bHasRowid ? -1 : 0); } } }else{ pCsr->iField++; if( pCsr->iField>0 ){ sqlite3_int64 iType; if( pCsr->pHdrPtr>=&pCsr->rec.aBuf[pCsr->nRec] || pCsr->iField>=DBDATA_MX_FIELD ){ bNextPage = 1; }else{ int szField = 0; pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType); szField = dbdataValueBytes(iType); if( (pCsr->nRec - (pCsr->pPtr - pCsr->rec.aBuf))<szField ){ pCsr->pPtr = &pCsr->rec.aBuf[pCsr->nRec]; }else{ pCsr->pPtr += szField; } } } } if( bNextPage ){ sqlite3_free(pCsr->aPage); pCsr->aPage = 0; pCsr->nRec = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; }else{ if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->rec.aBuf[pCsr->nHdr] ){ return SQLITE_OK; } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ pCsr->nRec = 0; pCsr->iCell++; } } } assert( !"can't get here" ); return SQLITE_OK; } /* ** Return true if the cursor is at EOF. */ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; return pCsr->aPage==0; } /* ** Return true if nul-terminated string zSchema ends in "()". Or false ** otherwise. */ static int dbdataIsFunction(const char *zSchema){ size_t n = strlen(zSchema); if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ return (int)n-2; } return 0; } /* ** Determine the size in pages of database zSchema (where zSchema is ** "main", "temp" or the name of an attached database) and set ** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, ** an SQLite error code. */ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; int rc, rc2; int nFunc = 0; sqlite3_stmt *pStmt = 0; if( (nFunc = dbdataIsFunction(zSchema))>0 ){ zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); }else{ zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); } if( zSql==0 ) return SQLITE_NOMEM; rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ pCsr->szDb = sqlite3_column_int(pStmt, 0); } rc2 = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ) rc = rc2; return rc; } /* ** Attempt to figure out the encoding of the database by retrieving page 1 ** and inspecting the header field. If successful, set the pCsr->enc variable ** and return SQLITE_OK. Otherwise, return an SQLite error code. */ static int dbdataGetEncoding(DbdataCursor *pCsr){ int rc = SQLITE_OK; int nPg1 = 0; u8 *aPg1 = 0; rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); if( rc==SQLITE_OK && nPg1>=(56+4) ){ pCsr->enc = get_uint32(&aPg1[56]); } sqlite3_free(aPg1); return rc; } /* ** xFilter method for sqlite_dbdata and sqlite_dbptr. */ static int dbdataFilter( sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; int rc = SQLITE_OK; const char *zSchema = "main"; (void)idxStr; (void)argc; dbdataResetCursor(pCsr); assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ zSchema = (const char*)sqlite3_value_text(argv[0]); if( zSchema==0 ) zSchema = ""; } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; }else{ rc = dbdataDbsize(pCsr, zSchema); } if( rc==SQLITE_OK ){ int nFunc = 0; if( pTab->pStmt ){ pCsr->pStmt = pTab->pStmt; pTab->pStmt = 0; }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); } }else{ rc = sqlite3_prepare_v2(pTab->db, "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, &pCsr->pStmt, 0 ); } } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); } /* Try to determine the encoding of the db by inspecting the header ** field on page 1. */ if( rc==SQLITE_OK ){ rc = dbdataGetEncoding(pCsr); } if( rc!=SQLITE_OK ){ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } return rc; } /* ** Return a column for the sqlite_dbdata or sqlite_dbptr table. */ static int dbdataColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, int i ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; if( pTab->bPtr ){ switch( i ){ case DBPTR_COLUMN_PGNO: sqlite3_result_int64(ctx, pCsr->iPgno); break; case DBPTR_COLUMN_CHILD: { int iOff = pCsr->iPgno==1 ? 100 : 0; if( pCsr->iCell<0 ){ iOff += 8; }else{ iOff += 12 + pCsr->iCell*2; if( iOff>pCsr->nPage ) return SQLITE_OK; iOff = get_uint16(&pCsr->aPage[iOff]); } if( iOff<=pCsr->nPage ){ sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); } break; } } }else{ switch( i ){ case DBDATA_COLUMN_PGNO: sqlite3_result_int64(ctx, pCsr->iPgno); break; case DBDATA_COLUMN_CELL: sqlite3_result_int(ctx, pCsr->iCell); break; case DBDATA_COLUMN_FIELD: sqlite3_result_int(ctx, pCsr->iField); break; case DBDATA_COLUMN_VALUE: { if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); }else if( &pCsr->rec.aBuf[pCsr->nRec] >= pCsr->pPtr ){ sqlite3_int64 iType; dbdataGetVarintU32(pCsr->pHdrPtr, &iType); dbdataValue( ctx, pCsr->enc, iType, pCsr->pPtr, &pCsr->rec.aBuf[pCsr->nRec] - pCsr->pPtr ); } break; } } } return SQLITE_OK; } /* ** Return the rowid for an sqlite_dbdata or sqlite_dptr table. */ static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; *pRowid = pCsr->iRowid; return SQLITE_OK; } /* ** Invoke this routine to register the "sqlite_dbdata" virtual table module */ static int sqlite3DbdataRegister(sqlite3 *db){ static sqlite3_module dbdata_module = { 0, /* iVersion */ 0, /* xCreate */ dbdataConnect, /* xConnect */ dbdataBestIndex, /* xBestIndex */ dbdataDisconnect, /* xDisconnect */ 0, /* xDestroy */ dbdataOpen, /* xOpen - open a cursor */ dbdataClose, /* xClose - close a cursor */ dbdataFilter, /* xFilter - configure scan constraints */ dbdataNext, /* xNext - advance a cursor */ dbdataEof, /* xEof - check for end of scan */ dbdataColumn, /* xColumn - read data */ dbdataRowid, /* xRowid - read data */ 0, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); if( rc==SQLITE_OK ){ rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); } return rc; } int sqlite3_dbdata_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ (void)pzErrMsg; return sqlite3DbdataRegister(db); } #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /************************* End ../ext/recover/dbdata.c ********************/ /************************* Begin ../ext/recover/sqlite3recover.c ******************/ /* ** 2022-08-27 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** */ /* #include "sqlite3recover.h" */ #include <assert.h> #include <string.h> #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Declaration for public API function in file dbdata.c. This may be called ** with NULL as the final two arguments to register the sqlite_dbptr and ** sqlite_dbdata virtual tables with a database handle. */ #ifdef _WIN32 #endif int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); /* typedef unsigned int u32; */ /* typedef unsigned char u8; */ /* typedef sqlite3_int64 i64; */ typedef struct RecoverTable RecoverTable; typedef struct RecoverColumn RecoverColumn; /* ** When recovering rows of data that can be associated with table ** definitions recovered from the sqlite_schema table, each table is ** represented by an instance of the following object. ** ** iRoot: ** The root page in the original database. Not necessarily (and usually ** not) the same in the recovered database. ** ** zTab: ** Name of the table. ** ** nCol/aCol[]: ** aCol[] is an array of nCol columns. In the order in which they appear ** in the table. ** ** bIntkey: ** Set to true for intkey tables, false for WITHOUT ROWID. ** ** iRowidBind: ** Each column in the aCol[] array has associated with it the index of ** the bind parameter its values will be bound to in the INSERT statement ** used to construct the output database. If the table does has a rowid ** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the ** index of the bind paramater to which the rowid value should be bound. ** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY ** KEY column, then the rowid value should be bound to the index associated ** with the column. ** ** pNext: ** All RecoverTable objects used by the recovery operation are allocated ** and populated as part of creating the recovered database schema in ** the output database, before any non-schema data are recovered. They ** are then stored in a singly-linked list linked by this variable beginning ** at sqlite3_recover.pTblList. */ struct RecoverTable { u32 iRoot; /* Root page in original database */ char *zTab; /* Name of table */ int nCol; /* Number of columns in table */ RecoverColumn *aCol; /* Array of columns */ int bIntkey; /* True for intkey, false for without rowid */ int iRowidBind; /* If >0, bind rowid to INSERT here */ RecoverTable *pNext; }; /* ** Each database column is represented by an instance of the following object ** stored in the RecoverTable.aCol[] array of the associated table. ** ** iField: ** The index of the associated field within database records. Or -1 if ** there is no associated field (e.g. for virtual generated columns). ** ** iBind: ** The bind index of the INSERT statement to bind this columns values ** to. Or 0 if there is no such index (iff (iField<0)). ** ** bIPK: ** True if this is the INTEGER PRIMARY KEY column. ** ** zCol: ** Name of column. ** ** eHidden: ** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). */ struct RecoverColumn { int iField; /* Field in record on disk */ int iBind; /* Binding to use in INSERT */ int bIPK; /* True for IPK column */ char *zCol; int eHidden; }; #define RECOVER_EHIDDEN_NONE … #define RECOVER_EHIDDEN_HIDDEN … #define RECOVER_EHIDDEN_VIRTUAL … #define RECOVER_EHIDDEN_STORED … /* ** Bitmap object used to track pages in the input database. Allocated ** and manipulated only by the following functions: ** ** recoverBitmapAlloc() ** recoverBitmapFree() ** recoverBitmapSet() ** recoverBitmapQuery() ** ** nPg: ** Largest page number that may be stored in the bitmap. The range ** of valid keys is 1 to nPg, inclusive. ** ** aElem[]: ** Array large enough to contain a bit for each key. For key value ** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. ** In other words, the following is true if bit iKey is set, or ** false if it is clear: ** ** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 */ typedef struct RecoverBitmap RecoverBitmap; struct RecoverBitmap { i64 nPg; /* Size of bitmap */ u32 aElem[1]; /* Array of 32-bit bitmasks */ }; /* ** State variables (part of the sqlite3_recover structure) used while ** recovering data for tables identified in the recovered schema (state ** RECOVER_STATE_WRITING). */ typedef struct RecoverStateW1 RecoverStateW1; struct RecoverStateW1 { sqlite3_stmt *pTbls; sqlite3_stmt *pSel; sqlite3_stmt *pInsert; int nInsert; RecoverTable *pTab; /* Table currently being written */ int nMax; /* Max column count in any schema table */ sqlite3_value **apVal; /* Array of nMax values */ int nVal; /* Number of valid entries in apVal[] */ int bHaveRowid; i64 iRowid; i64 iPrevPage; int iPrevCell; }; /* ** State variables (part of the sqlite3_recover structure) used while ** recovering data destined for the lost and found table (states ** RECOVER_STATE_LOSTANDFOUND[123]). */ typedef struct RecoverStateLAF RecoverStateLAF; struct RecoverStateLAF { RecoverBitmap *pUsed; i64 nPg; /* Size of db in pages */ sqlite3_stmt *pAllAndParent; sqlite3_stmt *pMapInsert; sqlite3_stmt *pMaxField; sqlite3_stmt *pUsedPages; sqlite3_stmt *pFindRoot; sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ sqlite3_stmt *pAllPage; sqlite3_stmt *pPageData; sqlite3_value **apVal; int nMaxField; }; /* ** Main recover handle structure. */ struct sqlite3_recover { /* Copies of sqlite3_recover_init[_sql]() parameters */ sqlite3 *dbIn; /* Input database */ char *zDb; /* Name of input db ("main" etc.) */ char *zUri; /* URI for output database */ void *pSqlCtx; /* SQL callback context */ int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ /* Values configured by sqlite3_recover_config() */ char *zStateDb; /* State database to use (or NULL) */ char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ int pgsz; int detected_pgsz; int nReserve; u8 *pPage1Disk; u8 *pPage1Cache; /* Error code and error message */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ int eState; int bCloseTransaction; /* Variables used with eState==RECOVER_STATE_WRITING */ RecoverStateW1 w1; /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ RecoverStateLAF laf; /* Fields used within sqlite3_recover_run() */ sqlite3 *dbOut; /* Output database */ sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ RecoverTable *pTblList; /* List of tables recovered from schema */ }; /* ** The various states in which an sqlite3_recover object may exist: ** ** RECOVER_STATE_INIT: ** The object is initially created in this state. sqlite3_recover_step() ** has yet to be called. This is the only state in which it is permitted ** to call sqlite3_recover_config(). ** ** RECOVER_STATE_WRITING: ** ** RECOVER_STATE_LOSTANDFOUND1: ** State to populate the bitmap of pages used by other tables or the ** database freelist. ** ** RECOVER_STATE_LOSTANDFOUND2: ** Populate the recovery.map table - used to figure out a "root" page ** for each lost page from in the database from which records are ** extracted. ** ** RECOVER_STATE_LOSTANDFOUND3: ** Populate the lost-and-found table itself. */ #define RECOVER_STATE_INIT … #define RECOVER_STATE_WRITING … #define RECOVER_STATE_LOSTANDFOUND1 … #define RECOVER_STATE_LOSTANDFOUND2 … #define RECOVER_STATE_LOSTANDFOUND3 … #define RECOVER_STATE_SCHEMA2 … #define RECOVER_STATE_DONE … /* ** Global variables used by this extension. */ typedef struct RecoverGlobal RecoverGlobal; struct RecoverGlobal { const sqlite3_io_methods *pMethods; sqlite3_recover *p; }; static RecoverGlobal recover_g; /* ** Use this static SQLite mutex to protect the globals during the ** first call to sqlite3_recover_step(). */ #define RECOVER_MUTEX_ID … /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). */ #define RECOVER_ROWID_DEFAULT … /* ** Mutex handling: ** ** recoverEnterMutex() - Enter the recovery mutex ** recoverLeaveMutex() - Leave the recovery mutex ** recoverAssertMutexHeld() - Assert that the recovery mutex is held */ #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 #define recoverEnterMutex … #define recoverLeaveMutex … #else static void recoverEnterMutex(void){ sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); } static void recoverLeaveMutex(void){ sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)); } #endif #if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG) static void recoverAssertMutexHeld(void){ assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) ); } #else #define recoverAssertMutexHeld … #endif /* ** Like strlen(). But handles NULL pointer arguments. */ static int recoverStrlen(const char *zStr){ if( zStr==0 ) return 0; return (int)(strlen(zStr)&0x7fffffff); } /* ** This function is a no-op if the recover handle passed as the first ** argument already contains an error (if p->errCode!=SQLITE_OK). ** ** Otherwise, an attempt is made to allocate, zero and return a buffer nByte ** bytes in size. If successful, a pointer to the new buffer is returned. Or, ** if an OOM error occurs, NULL is returned and the handle error code ** (p->errCode) set to SQLITE_NOMEM. */ static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ void *pRet = 0; assert( nByte>0 ); if( p->errCode==SQLITE_OK ){ pRet = sqlite3_malloc64(nByte); if( pRet ){ memset(pRet, 0, nByte); }else{ p->errCode = SQLITE_NOMEM; } } return pRet; } /* ** Set the error code and error message for the recover handle passed as ** the first argument. The error code is set to the value of parameter ** errCode. ** ** Parameter zFmt must be a printf() style formatting string. The handle ** error message is set to the result of using any trailing arguments for ** parameter substitutions in the formatting string. ** ** For example: ** ** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); */ static int recoverError( sqlite3_recover *p, int errCode, const char *zFmt, ... ){ char *z = 0; va_list ap; va_start(ap, zFmt); if( zFmt ){ z = sqlite3_vmprintf(zFmt, ap); va_end(ap); } sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; return errCode; } /* ** This function is a no-op if p->errCode is initially other than SQLITE_OK. ** In this case it returns NULL. ** ** Otherwise, an attempt is made to allocate and return a bitmap object ** large enough to store a bit for all page numbers between 1 and nPg, ** inclusive. The bitmap is initially zeroed. */ static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ int nElem = (nPg+1+31) / 32; int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); if( pRet ){ pRet->nPg = nPg; } return pRet; } /* ** Free a bitmap object allocated by recoverBitmapAlloc(). */ static void recoverBitmapFree(RecoverBitmap *pMap){ sqlite3_free(pMap); } /* ** Set the bit associated with page iPg in bitvec pMap. */ static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ if( iPg<=pMap->nPg ){ int iElem = (iPg / 32); int iBit = (iPg % 32); pMap->aElem[iElem] |= (((u32)1) << iBit); } } /* ** Query bitmap object pMap for the state of the bit associated with page ** iPg. Return 1 if it is set, or 0 otherwise. */ static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ int ret = 1; if( iPg<=pMap->nPg && iPg>0 ){ int iElem = (iPg / 32); int iBit = (iPg % 32); ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; } return ret; } /* ** Set the recover handle error to the error code and message returned by ** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database ** handle db. */ static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). ** ** Otherwise, it attempts to prepare the SQL statement in zSql against ** database handle db. If successful, the statement handle is returned. ** Or, if an error occurs, NULL is returned and an error left in the ** recover handle. */ static sqlite3_stmt *recoverPrepare( sqlite3_recover *p, sqlite3 *db, const char *zSql ){ sqlite3_stmt *pStmt = 0; if( p->errCode==SQLITE_OK ){ if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ recoverDbError(p, db); } } return pStmt; } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). ** ** Otherwise, argument zFmt is used as a printf() style format string, ** along with any trailing arguments, to create an SQL statement. This ** SQL statement is prepared against database handle db and, if successful, ** the statment handle returned. Or, if an error occurs - either during ** the printf() formatting or when preparing the resulting SQL - an ** error code and message are left in the recover handle. */ static sqlite3_stmt *recoverPreparePrintf( sqlite3_recover *p, sqlite3 *db, const char *zFmt, ... ){ sqlite3_stmt *pStmt = 0; if( p->errCode==SQLITE_OK ){ va_list ap; char *z; va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); if( z==0 ){ p->errCode = SQLITE_NOMEM; }else{ pStmt = recoverPrepare(p, db, z); sqlite3_free(z); } } return pStmt; } /* ** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() ** indicates that an error occurred, and there is not already an error ** in the recover handle passed as the first argument, set the error ** code and error message appropriately. ** ** This function returns a copy of the statement handle pointer passed ** as the second argument. */ static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ int rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){ recoverDbError(p, sqlite3_db_handle(pStmt)); } return pStmt; } /* ** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() ** indicates that an error occurred, and there is not already an error ** in the recover handle passed as the first argument, set the error ** code and error message appropriately. */ static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ sqlite3 *db = sqlite3_db_handle(pStmt); int rc = sqlite3_finalize(pStmt); if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ recoverDbError(p, db); } } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this ** case. ** ** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. ** Or, if an error occurs, leave an error code and message in the recover ** handle and return a copy of the error code. */ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ if( p->errCode==SQLITE_OK ){ int rc = sqlite3_exec(db, zSql, 0, 0, 0); if( rc ){ recoverDbError(p, db); } } return p->errCode; } /* ** Bind the value pVal to parameter iBind of statement pStmt. Leave an ** error in the recover handle passed as the first argument if an error ** (e.g. an OOM) occurs. */ static void recoverBindValue( sqlite3_recover *p, sqlite3_stmt *pStmt, int iBind, sqlite3_value *pVal ){ if( p->errCode==SQLITE_OK ){ int rc = sqlite3_bind_value(pStmt, iBind, pVal); if( rc ) recoverError(p, rc, 0); } } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). NULL is returned in this case. ** ** Otherwise, an attempt is made to interpret zFmt as a printf() style ** formatting string and the result of using the trailing arguments for ** parameter substitution with it written into a buffer obtained from ** sqlite3_malloc(). If successful, a pointer to the buffer is returned. ** It is the responsibility of the caller to eventually free the buffer ** using sqlite3_free(). ** ** Or, if an error occurs, an error code and message is left in the recover ** handle and NULL returned. */ static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ va_list ap; char *z; va_start(ap, zFmt); z = sqlite3_vmprintf(zFmt, ap); va_end(ap); if( p->errCode==SQLITE_OK ){ if( z==0 ) p->errCode = SQLITE_NOMEM; }else{ sqlite3_free(z); z = 0; } return z; } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). Zero is returned in this case. ** ** Otherwise, execute "PRAGMA page_count" against the input database. If ** successful, return the integer result. Or, if an error occurs, leave an ** error code and error message in the sqlite3_recover handle and return ** zero. */ static i64 recoverPageCount(sqlite3_recover *p){ i64 nPg = 0; if( p->errCode==SQLITE_OK ){ sqlite3_stmt *pStmt = 0; pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); if( pStmt ){ sqlite3_step(pStmt); nPg = sqlite3_column_int64(pStmt, 0); } recoverFinalize(p, pStmt); } return nPg; } /* ** Implementation of SQL scalar function "read_i32". The first argument to ** this function must be a blob. The second a non-negative integer. This ** function reads and returns a 32-bit big-endian integer from byte ** offset (4*<arg2>) of the blob. ** ** SELECT read_i32(<blob>, <idx>) */ static void recoverReadI32( sqlite3_context *context, int argc, sqlite3_value **argv ){ const unsigned char *pBlob; int nBlob; int iInt; assert( argc==2 ); nBlob = sqlite3_value_bytes(argv[0]); pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); iInt = sqlite3_value_int(argv[1]) & 0xFFFF; if( (iInt+1)*4<=nBlob ){ const unsigned char *a = &pBlob[iInt*4]; i64 iVal = ((i64)a[0]<<24) + ((i64)a[1]<<16) + ((i64)a[2]<< 8) + ((i64)a[3]<< 0); sqlite3_result_int64(context, iVal); } } /* ** Implementation of SQL scalar function "page_is_used". This function ** is used as part of the procedure for locating orphan rows for the ** lost-and-found table, and it depends on those routines having populated ** the sqlite3_recover.laf.pUsed variable. ** ** The only argument to this function is a page-number. It returns true ** if the page has already been used somehow during data recovery, or false ** otherwise. ** ** SELECT page_is_used(<pgno>); */ static void recoverPageIsUsed( sqlite3_context *pCtx, int nArg, sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); i64 pgno = sqlite3_value_int64(apArg[0]); assert( nArg==1 ); sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); } /* ** The implementation of a user-defined SQL function invoked by the ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages ** of the database being recovered. ** ** This function always takes a single integer argument. If the argument ** is zero, then the value returned is the number of pages in the db being ** recovered. If the argument is greater than zero, it is a page number. ** The value returned in this case is an SQL blob containing the data for ** the identified page of the db being recovered. e.g. ** ** SELECT getpage(0); -- return number of pages in db ** SELECT getpage(4); -- return page 4 of db as a blob of data */ static void recoverGetPage( sqlite3_context *pCtx, int nArg, sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); i64 pgno = sqlite3_value_int64(apArg[0]); sqlite3_stmt *pStmt = 0; assert( nArg==1 ); if( pgno==0 ){ i64 nPg = recoverPageCount(p); sqlite3_result_int64(pCtx, nPg); return; }else{ if( p->pGetPage==0 ){ pStmt = p->pGetPage = recoverPreparePrintf( p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb ); }else if( p->errCode==SQLITE_OK ){ pStmt = p->pGetPage; } if( pStmt ){ sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ const u8 *aPg; int nPg; assert( p->errCode==SQLITE_OK ); aPg = sqlite3_column_blob(pStmt, 0); nPg = sqlite3_column_bytes(pStmt, 0); if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ aPg = p->pPage1Disk; } sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); } recoverReset(p, pStmt); } } if( p->errCode ){ if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); sqlite3_result_error_code(pCtx, p->errCode); } } /* ** Find a string that is not found anywhere in z[]. Return a pointer ** to that string. ** ** Try to use zA and zB first. If both of those are already found in z[] ** then make up some string and store it in the buffer zBuf. */ static const char *recoverUnusedString( const char *z, /* Result must not appear anywhere in z */ const char *zA, const char *zB, /* Try these first */ char *zBuf /* Space to store a generated string */ ){ unsigned i = 0; if( strstr(z, zA)==0 ) return zA; if( strstr(z, zB)==0 ) return zB; do{ sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); }while( strstr(z,zBuf)!=0 ); return zBuf; } /* ** Implementation of scalar SQL function "escape_crnl". The argument passed to ** this function is the output of built-in function quote(). If the first ** character of the input is "'", indicating that the value passed to quote() ** was a text value, then this function searches the input for "\n" and "\r" ** characters and adds a wrapper similar to the following: ** ** replace(replace(<input>, '\n', char(10), '\r', char(13)); ** ** Or, if the first character of the input is not "'", then a copy of the input ** is returned. */ static void recoverEscapeCrnl( sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zText = (const char*)sqlite3_value_text(argv[0]); (void)argc; if( zText && zText[0]=='\'' ){ int nText = sqlite3_value_bytes(argv[0]); int i; char zBuf1[20]; char zBuf2[20]; const char *zNL = 0; const char *zCR = 0; int nCR = 0; int nNL = 0; for(i=0; zText[i]; i++){ if( zNL==0 && zText[i]=='\n' ){ zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); nNL = (int)strlen(zNL); } if( zCR==0 && zText[i]=='\r' ){ zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); nCR = (int)strlen(zCR); } } if( zNL || zCR ){ int iOut = 0; i64 nMax = (nNL > nCR) ? nNL : nCR; i64 nAlloc = nMax * nText + (nMax+64)*2; char *zOut = (char*)sqlite3_malloc64(nAlloc); if( zOut==0 ){ sqlite3_result_error_nomem(context); return; } if( zNL && zCR ){ memcpy(&zOut[iOut], "replace(replace(", 16); iOut += 16; }else{ memcpy(&zOut[iOut], "replace(", 8); iOut += 8; } for(i=0; zText[i]; i++){ if( zText[i]=='\n' ){ memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; }else if( zText[i]=='\r' ){ memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; }else{ zOut[iOut] = zText[i]; iOut++; } } if( zNL ){ memcpy(&zOut[iOut], ",'", 2); iOut += 2; memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; } if( zCR ){ memcpy(&zOut[iOut], ",'", 2); iOut += 2; memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; } sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); sqlite3_free(zOut); return; } } sqlite3_result_value(context, argv[0]); } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in ** this case. ** ** Otherwise, attempt to populate temporary table "recovery.schema" with the ** parts of the database schema that can be extracted from the input database. ** ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code ** and error message are left in the recover handle and a copy of the ** error code returned. It is not considered an error if part of all of ** the database schema cannot be recovered due to corruption. */ static int recoverCacheSchema(sqlite3_recover *p){ return recoverExec(p, p->dbOut, "WITH RECURSIVE pages(p) AS (" " SELECT 1" " UNION" " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" ")" "INSERT INTO recovery.schema SELECT" " max(CASE WHEN field=0 THEN value ELSE NULL END)," " max(CASE WHEN field=1 THEN value ELSE NULL END)," " max(CASE WHEN field=2 THEN value ELSE NULL END)," " max(CASE WHEN field=3 THEN value ELSE NULL END)," " max(CASE WHEN field=4 THEN value ELSE NULL END)" "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" " SELECT p FROM pages" ") GROUP BY pgno, cell" ); } /* ** If this recover handle is not in SQL callback mode (i.e. was not created ** using sqlite3_recover_init_sql()) of if an error has already occurred, ** this function is a no-op. Otherwise, issue a callback with SQL statement ** zSql as the parameter. ** ** If the callback returns non-zero, set the recover handle error code to ** the value returned (so that the caller will abandon processing). */ static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ if( p->errCode==SQLITE_OK && p->xSql ){ int res = p->xSql(p->pSqlCtx, zSql); if( res ){ recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); } } } /* ** Transfer the following settings from the input database to the output ** database: ** ** + page-size, ** + auto-vacuum settings, ** + database encoding, ** + user-version (PRAGMA user_version), and ** + application-id (PRAGMA application_id), and */ static void recoverTransferSettings(sqlite3_recover *p){ const char *aPragma[] = { "encoding", "page_size", "auto_vacuum", "user_version", "application_id" }; int ii; /* Truncate the output database to 0 pages in size. This is done by ** opening a new, empty, temp db, then using the backup API to clobber ** any existing output db with a copy of it. */ if( p->errCode==SQLITE_OK ){ sqlite3 *db2 = 0; int rc = sqlite3_open("", &db2); if( rc!=SQLITE_OK ){ recoverDbError(p, db2); return; } for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){ const char *zPrag = aPragma[ii]; sqlite3_stmt *p1 = 0; p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag); if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ const char *zArg = (const char*)sqlite3_column_text(p1, 0); char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); recoverSqlCallback(p, z2); recoverExec(p, db2, z2); sqlite3_free(z2); if( zArg==0 ){ recoverError(p, SQLITE_NOMEM, 0); } } recoverFinalize(p, p1); } recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); if( p->errCode==SQLITE_OK ){ sqlite3 *db = p->dbOut; sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); if( pBackup ){ sqlite3_backup_step(pBackup, -1); p->errCode = sqlite3_backup_finish(pBackup); }else{ recoverDbError(p, db); } } sqlite3_close(db2); } } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in ** this case. ** ** Otherwise, an attempt is made to open the output database, attach ** and create the schema of the temporary database used to store ** intermediate data, and to register all required user functions and ** virtual table modules with the output handle. ** ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code ** and error message are left in the recover handle and a copy of the ** error code returned. */ static int recoverOpenOutput(sqlite3_recover *p){ struct Func { const char *zName; int nArg; void (*xFunc)(sqlite3_context*,int,sqlite3_value **); } aFunc[] = { { "getpage", 1, recoverGetPage }, { "page_is_used", 1, recoverPageIsUsed }, { "read_i32", 2, recoverReadI32 }, { "escape_crnl", 1, recoverEscapeCrnl }, }; const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; sqlite3 *db = 0; /* New database handle */ int ii; /* For iterating through aFunc[] */ assert( p->dbOut==0 ); if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ recoverDbError(p, db); } /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. ** These two are registered with the output database handle - this ** module depends on the input handle supporting the sqlite_dbpage ** virtual table only. */ if( p->errCode==SQLITE_OK ){ p->errCode = sqlite3_dbdata_init(db, 0, 0); } /* Register the custom user-functions with the output handle. */ for(ii=0; p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0])); ii++){ p->errCode = sqlite3_create_function(db, aFunc[ii].zName, aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 ); } p->dbOut = db; return p->errCode; } /* ** Attach the auxiliary database 'recovery' to the output database handle. ** This temporary database is used during the recovery process and then ** discarded. */ static void recoverOpenRecovery(sqlite3_recover *p){ char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); recoverExec(p, p->dbOut, zSql); recoverExec(p, p->dbOut, "PRAGMA writable_schema = 1;" "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" ); sqlite3_free(zSql); } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). ** ** Otherwise, argument zName must be the name of a table that has just been ** created in the output database. This function queries the output db ** for the schema of said table, and creates a RecoverTable object to ** store the schema in memory. The new RecoverTable object is linked into ** the list at sqlite3_recover.pTblList. ** ** Parameter iRoot must be the root page of table zName in the INPUT ** database. */ static void recoverAddTable( sqlite3_recover *p, const char *zName, /* Name of table created in output db */ i64 iRoot /* Root page of same table in INPUT db */ ){ sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA table_xinfo(%Q)", zName ); if( pStmt ){ int iPk = -1; int iBind = 1; RecoverTable *pNew = 0; int nCol = 0; int nName = recoverStrlen(zName); int nByte = 0; while( sqlite3_step(pStmt)==SQLITE_ROW ){ nCol++; nByte += (sqlite3_column_bytes(pStmt, 1)+1); } nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; recoverReset(p, pStmt); pNew = recoverMalloc(p, nByte); if( pNew ){ int i = 0; int iField = 0; char *csr = 0; pNew->aCol = (RecoverColumn*)&pNew[1]; pNew->zTab = csr = (char*)&pNew->aCol[nCol]; pNew->nCol = nCol; pNew->iRoot = iRoot; memcpy(csr, zName, nName); csr += nName+1; for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ int iPKF = sqlite3_column_int(pStmt, 5); int n = sqlite3_column_bytes(pStmt, 1); const char *z = (const char*)sqlite3_column_text(pStmt, 1); const char *zType = (const char*)sqlite3_column_text(pStmt, 2); int eHidden = sqlite3_column_int(pStmt, 6); if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; if( iPKF>1 ) iPk = -2; pNew->aCol[i].zCol = csr; pNew->aCol[i].eHidden = eHidden; if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ pNew->aCol[i].iField = -1; }else{ pNew->aCol[i].iField = iField++; } if( eHidden!=RECOVER_EHIDDEN_VIRTUAL && eHidden!=RECOVER_EHIDDEN_STORED ){ pNew->aCol[i].iBind = iBind++; } memcpy(csr, z, n); csr += (n+1); } pNew->pNext = p->pTblList; p->pTblList = pNew; pNew->bIntkey = 1; } recoverFinalize(p, pStmt); pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ int iField = sqlite3_column_int(pStmt, 0); int iCol = sqlite3_column_int(pStmt, 1); assert( iCol<pNew->nCol ); pNew->aCol[iCol].iField = iField; pNew->bIntkey = 0; iPk = -2; } recoverFinalize(p, pStmt); if( p->errCode==SQLITE_OK ){ if( iPk>=0 ){ pNew->aCol[iPk].bIPK = 1; }else if( pNew->bIntkey ){ pNew->iRowidBind = iBind++; } } } } /* ** This function is called after recoverCacheSchema() has cached those parts ** of the input database schema that could be recovered in temporary table ** "recovery.schema". This function creates in the output database copies ** of all parts of that schema that must be created before the tables can ** be populated. Specifically, this means: ** ** * all tables that are not VIRTUAL, and ** * UNIQUE indexes. ** ** If the recovery handle uses SQL callbacks, then callbacks containing ** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. ** ** Additionally, records are added to the sqlite_schema table of the ** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE ** records are written directly to sqlite_schema, not actually executed. ** If the handle is in SQL callback mode, then callbacks are invoked ** with equivalent SQL statements. */ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; sqlite3_stmt *pTblname = 0; pSelect = recoverPrepare(p, p->dbOut, "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" " SELECT rootpage, name, sql, " " type='table', " " sql LIKE 'create virtual%'," " (type='index' AND (sql LIKE '%unique%' OR ?1))" " FROM recovery.schema" ")" "SELECT rootpage, tbl, isVirtual, name, sql" " FROM dbschema " " WHERE tbl OR isIndex" " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" ); pTblname = recoverPrepare(p, p->dbOut, "SELECT name FROM sqlite_schema " "WHERE type='table' ORDER BY rowid DESC LIMIT 1" ); if( pSelect ){ sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); while( sqlite3_step(pSelect)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(pSelect, 0); int bTable = sqlite3_column_int(pSelect, 1); int bVirtual = sqlite3_column_int(pSelect, 2); const char *zName = (const char*)sqlite3_column_text(pSelect, 3); const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); char *zFree = 0; int rc = SQLITE_OK; if( bVirtual ){ zSql = (const char*)(zFree = recoverMPrintf(p, "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", zName, zName, zSql )); } rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); if( zTbl ) recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } }else if( rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); } sqlite3_free(zFree); } } recoverFinalize(p, pSelect); recoverFinalize(p, pTblname); return p->errCode; } /* ** This function is called after the output database has been populated. It ** adds all recovered schema elements that were not created in the output ** database by recoverWriteSchema1() - everything except for tables and ** UNIQUE indexes. Specifically: ** ** * views, ** * triggers, ** * non-UNIQUE indexes. ** ** If the recover handle is in SQL callback mode, then equivalent callbacks ** are issued to create the schema elements. */ static int recoverWriteSchema2(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; pSelect = recoverPrepare(p, p->dbOut, p->bSlowIndexes ? "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND type!='index'" : "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" ); if( pSelect ){ while( sqlite3_step(pSelect)==SQLITE_ROW ){ const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); }else if( rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); } } } recoverFinalize(p, pSelect); return p->errCode; } /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). In this case it returns NULL. ** ** Otherwise, if the recover handle is configured to create an output ** database (was created by sqlite3_recover_init()), then this function ** prepares and returns an SQL statement to INSERT a new record into table ** pTab, assuming the first nField fields of a record extracted from disk ** are valid. ** ** For example, if table pTab is: ** ** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); ** ** And nField is 4, then the SQL statement prepared and returned is: ** ** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); ** ** In this case even though 4 values were extracted from the input db, ** only 3 are written to the output, as the generated STORED column ** cannot be written. ** ** If the recover handle is in SQL callback mode, then the SQL statement ** prepared is such that evaluating it returns a single row containing ** a single text value - itself an SQL statement similar to the above, ** except with SQL literals in place of the variables. For example: ** ** SELECT 'INSERT INTO (a, c, d) VALUES (' ** || quote(?1) || ', ' ** || quote(?2) || ', ' ** || quote(?3) || ')'; ** ** In either case, it is the responsibility of the caller to eventually ** free the statement handle using sqlite3_finalize(). */ static sqlite3_stmt *recoverInsertStmt( sqlite3_recover *p, RecoverTable *pTab, int nField ){ sqlite3_stmt *pRet = 0; const char *zSep = ""; const char *zSqlSep = ""; char *zSql = 0; char *zFinal = 0; char *zBind = 0; int ii; int bSql = p->xSql ? 1 : 0; if( nField<=0 ) return 0; assert( nField<=pTab->nCol ); zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); if( pTab->iRowidBind ){ assert( pTab->bIntkey ); zSql = recoverMPrintf(p, "%z_rowid_", zSql); if( bSql ){ zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); }else{ zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); } zSqlSep = "||', '||"; zSep = ", "; } for(ii=0; ii<nField; ii++){ int eHidden = pTab->aCol[ii].eHidden; if( eHidden!=RECOVER_EHIDDEN_VIRTUAL && eHidden!=RECOVER_EHIDDEN_STORED ){ assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); if( bSql ){ zBind = recoverMPrintf(p, "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind ); zSqlSep = "||', '||"; }else{ zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); } zSep = ", "; } } if( bSql ){ zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", zSql, zBind ); }else{ zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); } pRet = recoverPrepare(p, p->dbOut, zFinal); sqlite3_free(zSql); sqlite3_free(zBind); sqlite3_free(zFinal); return pRet; } /* ** Search the list of RecoverTable objects at p->pTblList for one that ** has root page iRoot in the input database. If such an object is found, ** return a pointer to it. Otherwise, return NULL. */ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ RecoverTable *pRet = 0; for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); return pRet; } /* ** This function attempts to create a lost and found table within the ** output db. If successful, it returns a pointer to a buffer containing ** the name of the new table. It is the responsibility of the caller to ** eventually free this buffer using sqlite3_free(). ** ** If an error occurs, NULL is returned and an error code and error ** message left in the recover handle. */ static char *recoverLostAndFoundCreate( sqlite3_recover *p, /* Recover object */ int nField /* Number of column fields in new table */ ){ char *zTbl = 0; sqlite3_stmt *pProbe = 0; int ii = 0; pProbe = recoverPrepare(p, p->dbOut, "SELECT 1 FROM sqlite_schema WHERE name=?" ); for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ int bFail = 0; if( ii<0 ){ zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); }else{ zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); } if( p->errCode==SQLITE_OK ){ sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pProbe) ){ bFail = 1; } recoverReset(p, pProbe); } if( bFail ){ sqlite3_clear_bindings(pProbe); sqlite3_free(zTbl); zTbl = 0; } } recoverFinalize(p, pProbe); if( zTbl ){ const char *zSep = 0; char *zField = 0; char *zSql = 0; zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; for(ii=0; p->errCode==SQLITE_OK && ii<nField; ii++){ zField = recoverMPrintf(p, "%z%sc%d", zField, zSep, ii); zSep = ", "; } zSql = recoverMPrintf(p, "CREATE TABLE %s(%s)", zTbl, zField); sqlite3_free(zField); recoverExec(p, p->dbOut, zSql); recoverSqlCallback(p, zSql); sqlite3_free(zSql); }else if( p->errCode==SQLITE_OK ){ recoverError( p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound ); } return zTbl; } /* ** Synthesize and prepare an INSERT statement to write to the lost_and_found ** table in the output database. The name of the table is zTab, and it has ** nField c* fields. */ static sqlite3_stmt *recoverLostAndFoundInsert( sqlite3_recover *p, const char *zTab, int nField ){ int nTotal = nField + 4; int ii; char *zBind = 0; sqlite3_stmt *pRet = 0; if( p->xSql==0 ){ for(ii=0; ii<nTotal; ii++){ zBind = recoverMPrintf(p, "%z%s?", zBind, zBind?", ":"", ii); } pRet = recoverPreparePrintf( p, p->dbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind ); }else{ const char *zSep = ""; for(ii=0; ii<nTotal; ii++){ zBind = recoverMPrintf(p, "%z%squote(?)", zBind, zSep); zSep = "|| ', ' ||"; } pRet = recoverPreparePrintf( p, p->dbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind ); } sqlite3_free(zBind); return pRet; } /* ** Input database page iPg contains data that will be written to the ** lost-and-found table of the output database. This function attempts ** to identify the root page of the tree that page iPg belonged to. ** If successful, it sets output variable (*piRoot) to the page number ** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, ** an SQLite error code is returned and the final value of *piRoot ** undefined. */ static int recoverLostAndFoundFindRoot( sqlite3_recover *p, i64 iPg, i64 *piRoot ){ RecoverStateLAF *pLaf = &p->laf; if( pLaf->pFindRoot==0 ){ pLaf->pFindRoot = recoverPrepare(p, p->dbOut, "WITH RECURSIVE p(pgno) AS (" " SELECT ?" " UNION" " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" ") " "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " " AND m.parent IS NULL" ); } if( p->errCode==SQLITE_OK ){ sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); }else{ *piRoot = iPg; } recoverReset(p, pLaf->pFindRoot); } return p->errCode; } /* ** Recover data from page iPage of the input database and write it to ** the lost-and-found table in the output database. */ static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ RecoverStateLAF *pLaf = &p->laf; sqlite3_value **apVal = pLaf->apVal; sqlite3_stmt *pPageData = pLaf->pPageData; sqlite3_stmt *pInsert = pLaf->pInsert; int nVal = -1; int iPrevCell = 0; i64 iRoot = 0; int bHaveRowid = 0; i64 iRowid = 0; int ii = 0; if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; sqlite3_bind_int64(pPageData, 1, iPage); while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ int iCell = sqlite3_column_int64(pPageData, 0); int iField = sqlite3_column_int64(pPageData, 1); if( iPrevCell!=iCell && nVal>=0 ){ /* Insert the new row */ sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ if( bHaveRowid ){ sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ } for(ii=0; ii<nVal; ii++){ recoverBindValue(p, pInsert, 5+ii, apVal[ii]); } if( sqlite3_step(pInsert)==SQLITE_ROW ){ recoverSqlCallback(p, (const char*)sqlite3_column_text(pInsert, 0)); } recoverReset(p, pInsert); /* Discard the accumulated row data */ for(ii=0; ii<nVal; ii++){ sqlite3_value_free(apVal[ii]); apVal[ii] = 0; } sqlite3_clear_bindings(pInsert); bHaveRowid = 0; nVal = -1; } if( iCell<0 ) break; if( iField<0 ){ assert( nVal==-1 ); iRowid = sqlite3_column_int64(pPageData, 2); bHaveRowid = 1; nVal = 0; }else if( iField<pLaf->nMaxField ){ sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); apVal[iField] = sqlite3_value_dup(pVal); assert( iField==nVal || (nVal==-1 && iField==0) ); nVal = iField+1; if( apVal[iField]==0 ){ recoverError(p, SQLITE_NOMEM, 0); } } iPrevCell = iCell; } recoverReset(p, pPageData); for(ii=0; ii<nVal; ii++){ sqlite3_value_free(apVal[ii]); apVal[ii] = 0; } } /* ** Perform one step (sqlite3_recover_step()) of work for the connection ** passed as the only argument, which is guaranteed to be in ** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found ** table of the output database is populated with recovered data that can ** not be assigned to any recovered schema object. */ static int recoverLostAndFound3Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; if( p->errCode==SQLITE_OK ){ if( pLaf->pInsert==0 ){ return SQLITE_DONE; }else{ if( p->errCode==SQLITE_OK ){ int res = sqlite3_step(pLaf->pAllPage); if( res==SQLITE_ROW ){ i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ recoverLostAndFoundOnePage(p, iPage); } }else{ recoverReset(p, pLaf->pAllPage); return SQLITE_DONE; } } } } return SQLITE_OK; } /* ** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 ** state - during which the lost-and-found table of the output database ** is populated with recovered data that can not be assigned to any ** recovered schema object. */ static void recoverLostAndFound3Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; if( pLaf->nMaxField>0 ){ char *zTab = 0; /* Name of lost_and_found table */ zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); sqlite3_free(zTab); pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, "WITH RECURSIVE seq(ii) AS (" " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" ")" "SELECT ii FROM seq" , p->laf.nPg ); pLaf->pPageData = recoverPrepare(p, p->dbOut, "SELECT cell, field, value " "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " "UNION ALL " "SELECT -1, -1, -1" ); pLaf->apVal = (sqlite3_value**)recoverMalloc(p, pLaf->nMaxField*sizeof(sqlite3_value*) ); } } /* ** Initialize resources required in RECOVER_STATE_WRITING state - during which ** tables recovered from the schema of the input database are populated with ** recovered data. */ static int recoverWriteDataInit(sqlite3_recover *p){ RecoverStateW1 *p1 = &p->w1; RecoverTable *pTbl = 0; int nByte = 0; /* Figure out the maximum number of columns for any table in the schema */ assert( p1->nMax==0 ); for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; } /* Allocate an array of (sqlite3_value*) in which to accumulate the values ** that will be written to the output database in a single row. */ nByte = sizeof(sqlite3_value*) * (p1->nMax+1); p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); if( p1->apVal==0 ) return p->errCode; /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT ** to loop through cells that appear to belong to a single table (pSel). */ p1->pTbls = recoverPrepare(p, p->dbOut, "SELECT rootpage FROM recovery.schema " " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" " ORDER BY (tbl_name='sqlite_sequence') ASC" ); p1->pSel = recoverPrepare(p, p->dbOut, "WITH RECURSIVE pages(page) AS (" " SELECT ?1" " UNION" " SELECT child FROM sqlite_dbptr('getpage()'), pages " " WHERE pgno=page" ") " "SELECT page, cell, field, value " "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " "UNION ALL " "SELECT 0, 0, 0, 0" ); return p->errCode; } /* ** Clean up resources allocated by recoverWriteDataInit() (stuff in ** sqlite3_recover.w1). */ static void recoverWriteDataCleanup(sqlite3_recover *p){ RecoverStateW1 *p1 = &p->w1; int ii; for(ii=0; ii<p1->nVal; ii++){ sqlite3_value_free(p1->apVal[ii]); } sqlite3_free(p1->apVal); recoverFinalize(p, p1->pInsert); recoverFinalize(p, p1->pTbls); recoverFinalize(p, p1->pSel); memset(p1, 0, sizeof(*p1)); } /* ** Perform one step (sqlite3_recover_step()) of work for the connection ** passed as the only argument, which is guaranteed to be in ** RECOVER_STATE_WRITING state - during which tables recovered from the ** schema of the input database are populated with recovered data. */ static int recoverWriteDataStep(sqlite3_recover *p){ RecoverStateW1 *p1 = &p->w1; sqlite3_stmt *pSel = p1->pSel; sqlite3_value **apVal = p1->apVal; if( p->errCode==SQLITE_OK && p1->pTab==0 ){ if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); p1->pTab = recoverFindTable(p, iRoot); recoverFinalize(p, p1->pInsert); p1->pInsert = 0; /* If this table is unknown, return early. The caller will invoke this ** function again and it will move on to the next table. */ if( p1->pTab==0 ) return p->errCode; /* If this is the sqlite_sequence table, delete any rows added by ** earlier INSERT statements on tables with AUTOINCREMENT primary ** keys before recovering its contents. The p1->pTbls SELECT statement ** is rigged to deliver "sqlite_sequence" last of all, so we don't ** worry about it being modified after it is recovered. */ if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); } /* Bind the root page of this table within the original database to ** SELECT statement p1->pSel. The SELECT statement will then iterate ** through cells that look like they belong to table pTab. */ sqlite3_bind_int64(pSel, 1, iRoot); p1->nVal = 0; p1->bHaveRowid = 0; p1->iPrevPage = -1; p1->iPrevCell = -1; }else{ return SQLITE_DONE; } } assert( p->errCode!=SQLITE_OK || p1->pTab ); if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ RecoverTable *pTab = p1->pTab; i64 iPage = sqlite3_column_int64(pSel, 0); int iCell = sqlite3_column_int(pSel, 1); int iField = sqlite3_column_int(pSel, 2); sqlite3_value *pVal = sqlite3_column_value(pSel, 3); int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); assert( bNewCell==0 || (iField==-1 || iField==0) ); assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); if( bNewCell ){ int ii = 0; if( p1->nVal>=0 ){ if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ recoverFinalize(p, p1->pInsert); p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); p1->nInsert = p1->nVal; } if( p1->nVal>0 ){ sqlite3_stmt *pInsert = p1->pInsert; for(ii=0; ii<pTab->nCol; ii++){ RecoverColumn *pCol = &pTab->aCol[ii]; int iBind = pCol->iBind; if( iBind>0 ){ if( pCol->bIPK ){ sqlite3_bind_int64(pInsert, iBind, p1->iRowid); }else if( pCol->iField<p1->nVal ){ recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); } } } if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); } if( SQLITE_ROW==sqlite3_step(pInsert) ){ const char *z = (const char*)sqlite3_column_text(pInsert, 0); recoverSqlCallback(p, z); } recoverReset(p, pInsert); assert( p->errCode || pInsert ); if( pInsert ) sqlite3_clear_bindings(pInsert); } } for(ii=0; ii<p1->nVal; ii++){ sqlite3_value_free(apVal[ii]); apVal[ii] = 0; } p1->nVal = -1; p1->bHaveRowid = 0; } if( iPage!=0 ){ if( iField<0 ){ p1->iRowid = sqlite3_column_int64(pSel, 3); assert( p1->nVal==-1 ); p1->nVal = 0; p1->bHaveRowid = 1; }else if( iField<pTab->nCol ){ assert( apVal[iField]==0 ); apVal[iField] = sqlite3_value_dup( pVal ); if( apVal[iField]==0 ){ recoverError(p, SQLITE_NOMEM, 0); } p1->nVal = iField+1; } p1->iPrevCell = iCell; p1->iPrevPage = iPage; } }else{ recoverReset(p, pSel); p1->pTab = 0; } return p->errCode; } /* ** Initialize resources required by sqlite3_recover_step() in ** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not ** already allocated to a recovered schema element is determined. */ static void recoverLostAndFound1Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; sqlite3_stmt *pStmt = 0; assert( p->laf.pUsed==0 ); pLaf->nPg = recoverPageCount(p); pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); /* Prepare a statement to iterate through all pages that are part of any tree ** in the recoverable part of the input database schema to the bitmap. And, ** if !p->bFreelistCorrupt, add all pages that appear to be part of the ** freelist. */ pStmt = recoverPrepare( p, p->dbOut, "WITH trunk(pgno) AS (" " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" " UNION" " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" ")," "trunkdata(pgno, data) AS (" " SELECT pgno, getpage(pgno) FROM trunk" ")," "freelist(data, n, freepgno) AS (" " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" " UNION ALL" " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" ")," "" "roots(r) AS (" " SELECT 1 UNION ALL" " SELECT rootpage FROM recovery.schema WHERE rootpage>0" ")," "used(page) AS (" " SELECT r FROM roots" " UNION" " SELECT child FROM sqlite_dbptr('getpage()'), used " " WHERE pgno=page" ") " "SELECT page FROM used" " UNION ALL " "SELECT freepgno FROM freelist WHERE NOT ?" ); if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); pLaf->pUsedPages = pStmt; } /* ** Perform one step (sqlite3_recover_step()) of work for the connection ** passed as the only argument, which is guaranteed to be in ** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not ** already allocated to a recovered schema element is determined. */ static int recoverLostAndFound1Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; int rc = p->errCode; if( rc==SQLITE_OK ){ rc = sqlite3_step(pLaf->pUsedPages); if( rc==SQLITE_ROW ){ i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); recoverBitmapSet(pLaf->pUsed, iPg); rc = SQLITE_OK; }else{ recoverFinalize(p, pLaf->pUsedPages); pLaf->pUsedPages = 0; } } return rc; } /* ** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 ** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 ** are sorted into sets that likely belonged to the same database tree. */ static void recoverLostAndFound2Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; assert( p->laf.pAllAndParent==0 ); assert( p->laf.pMapInsert==0 ); assert( p->laf.pMaxField==0 ); assert( p->laf.nMaxField==0 ); pLaf->pMapInsert = recoverPrepare(p, p->dbOut, "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" ); pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, "WITH RECURSIVE seq(ii) AS (" " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" ")" "SELECT pgno, child FROM sqlite_dbptr('getpage()') " " UNION ALL " "SELECT NULL, ii FROM seq", p->laf.nPg ); pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" ); } /* ** Perform one step (sqlite3_recover_step()) of work for the connection ** passed as the only argument, which is guaranteed to be in ** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified ** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged ** to the same database tree. */ static int recoverLostAndFound2Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; if( p->errCode==SQLITE_OK ){ int res = sqlite3_step(pLaf->pAllAndParent); if( res==SQLITE_ROW ){ i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); sqlite3_bind_value(pLaf->pMapInsert, 2, sqlite3_column_value(pLaf->pAllAndParent, 0) ); sqlite3_step(pLaf->pMapInsert); recoverReset(p, pLaf->pMapInsert); sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ int nMax = sqlite3_column_int(pLaf->pMaxField, 0); if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; } recoverReset(p, pLaf->pMaxField); } }else{ recoverFinalize(p, pLaf->pAllAndParent); pLaf->pAllAndParent =0; return SQLITE_DONE; } } return p->errCode; } /* ** Free all resources allocated as part of sqlite3_recover_step() calls ** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. */ static void recoverLostAndFoundCleanup(sqlite3_recover *p){ recoverBitmapFree(p->laf.pUsed); p->laf.pUsed = 0; sqlite3_finalize(p->laf.pUsedPages); sqlite3_finalize(p->laf.pAllAndParent); sqlite3_finalize(p->laf.pMapInsert); sqlite3_finalize(p->laf.pMaxField); sqlite3_finalize(p->laf.pFindRoot); sqlite3_finalize(p->laf.pInsert); sqlite3_finalize(p->laf.pAllPage); sqlite3_finalize(p->laf.pPageData); p->laf.pUsedPages = 0; p->laf.pAllAndParent = 0; p->laf.pMapInsert = 0; p->laf.pMaxField = 0; p->laf.pFindRoot = 0; p->laf.pInsert = 0; p->laf.pAllPage = 0; p->laf.pPageData = 0; sqlite3_free(p->laf.apVal); p->laf.apVal = 0; } /* ** Free all resources allocated as part of sqlite3_recover_step() calls. */ static void recoverFinalCleanup(sqlite3_recover *p){ RecoverTable *pTab = 0; RecoverTable *pNext = 0; recoverWriteDataCleanup(p); recoverLostAndFoundCleanup(p); for(pTab=p->pTblList; pTab; pTab=pNext){ pNext = pTab->pNext; sqlite3_free(pTab); } p->pTblList = 0; sqlite3_finalize(p->pGetPage); p->pGetPage = 0; sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); { #ifndef NDEBUG int res = #endif sqlite3_close(p->dbOut); assert( res==SQLITE_OK ); } p->dbOut = 0; } /* ** Decode and return an unsigned 16-bit big-endian integer value from ** buffer a[]. */ static u32 recoverGetU16(const u8 *a){ return (((u32)a[0])<<8) + ((u32)a[1]); } /* ** Decode and return an unsigned 32-bit big-endian integer value from ** buffer a[]. */ static u32 recoverGetU32(const u8 *a){ return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); } /* ** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) ** and return the number of bytes consumed. */ static int recoverGetVarint(const u8 *a, i64 *pVal){ sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ u = (u<<7) + (a[i]&0x7f); if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } u = (u<<8) + (a[i]&0xff); *pVal = (sqlite3_int64)u; return 9; } /* ** The second argument points to a buffer n bytes in size. If this buffer ** or a prefix thereof appears to contain a well-formed SQLite b-tree page, ** return the page-size in bytes. Otherwise, if the buffer does not ** appear to contain a well-formed b-tree page, return 0. */ static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ u8 *aUsed = aTmp; int nFrag = 0; int nActual = 0; int iFree = 0; int nCell = 0; /* Number of cells on page */ int iCellOff = 0; /* Offset of cell array in page */ int iContent = 0; int eType = 0; int ii = 0; eType = (int)a[0]; if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; iFree = (int)recoverGetU16(&a[1]); nCell = (int)recoverGetU16(&a[3]); iContent = (int)recoverGetU16(&a[5]); if( iContent==0 ) iContent = 65536; nFrag = (int)a[7]; if( iContent>n ) return 0; memset(aUsed, 0, n); memset(aUsed, 0xFF, iContent); /* Follow the free-list. This is the same format for all b-tree pages. */ if( iFree && iFree<=iContent ) return 0; while( iFree ){ int iNext = 0; int nByte = 0; if( iFree>(n-4) ) return 0; iNext = recoverGetU16(&a[iFree]); nByte = recoverGetU16(&a[iFree+2]); if( iFree+nByte>n || nByte<4 ) return 0; if( iNext && iNext<iFree+nByte ) return 0; memset(&aUsed[iFree], 0xFF, nByte); iFree = iNext; } /* Run through the cells */ if( eType==0x02 || eType==0x05 ){ iCellOff = 12; }else{ iCellOff = 8; } if( (iCellOff + 2*nCell)>iContent ) return 0; for(ii=0; ii<nCell; ii++){ int iByte; i64 nPayload = 0; int nByte = 0; int iOff = recoverGetU16(&a[iCellOff + 2*ii]); if( iOff<iContent || iOff>n ){ return 0; } if( eType==0x05 || eType==0x02 ) nByte += 4; nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); if( eType==0x0D ){ i64 dummy = 0; nByte += recoverGetVarint(&a[iOff+nByte], &dummy); } if( eType!=0x05 ){ int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); int M = ((n-12)*32/255)-23; int K = M+((nPayload-M)%(n-4)); if( nPayload<X ){ nByte += nPayload; }else if( K<=X ){ nByte += K+4; }else{ nByte += M+4; } } if( iOff+nByte>n ){ return 0; } for(iByte=iOff; iByte<(iOff+nByte); iByte++){ if( aUsed[iByte]!=0 ){ return 0; } aUsed[iByte] = 0xFF; } } nActual = 0; for(ii=0; ii<n; ii++){ if( aUsed[ii]==0 ) nActual++; } return (nActual==nFrag); } static int recoverVfsClose(sqlite3_file*); static int recoverVfsRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); static int recoverVfsWrite(sqlite3_file*, const void*, int, sqlite3_int64); static int recoverVfsTruncate(sqlite3_file*, sqlite3_int64 size); static int recoverVfsSync(sqlite3_file*, int flags); static int recoverVfsFileSize(sqlite3_file*, sqlite3_int64 *pSize); static int recoverVfsLock(sqlite3_file*, int); static int recoverVfsUnlock(sqlite3_file*, int); static int recoverVfsCheckReservedLock(sqlite3_file*, int *pResOut); static int recoverVfsFileControl(sqlite3_file*, int op, void *pArg); static int recoverVfsSectorSize(sqlite3_file*); static int recoverVfsDeviceCharacteristics(sqlite3_file*); static int recoverVfsShmMap(sqlite3_file*, int, int, int, void volatile**); static int recoverVfsShmLock(sqlite3_file*, int offset, int n, int flags); static void recoverVfsShmBarrier(sqlite3_file*); static int recoverVfsShmUnmap(sqlite3_file*, int deleteFlag); static int recoverVfsFetch(sqlite3_file*, sqlite3_int64, int, void**); static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p); static sqlite3_io_methods recover_methods = { 2, /* iVersion */ recoverVfsClose, recoverVfsRead, recoverVfsWrite, recoverVfsTruncate, recoverVfsSync, recoverVfsFileSize, recoverVfsLock, recoverVfsUnlock, recoverVfsCheckReservedLock, recoverVfsFileControl, recoverVfsSectorSize, recoverVfsDeviceCharacteristics, recoverVfsShmMap, recoverVfsShmLock, recoverVfsShmBarrier, recoverVfsShmUnmap, recoverVfsFetch, recoverVfsUnfetch }; static int recoverVfsClose(sqlite3_file *pFd){ assert( pFd->pMethods!=&recover_methods ); return pFd->pMethods->xClose(pFd); } /* ** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. */ static void recoverPutU16(u8 *a, u32 v){ a[0] = (v>>8) & 0x00FF; a[1] = (v>>0) & 0x00FF; } /* ** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. */ static void recoverPutU32(u8 *a, u32 v){ a[0] = (v>>24) & 0x00FF; a[1] = (v>>16) & 0x00FF; a[2] = (v>>8) & 0x00FF; a[3] = (v>>0) & 0x00FF; } /* ** Detect the page-size of the database opened by file-handle pFd by ** searching the first part of the file for a well-formed SQLite b-tree ** page. If parameter nReserve is non-zero, then as well as searching for ** a b-tree page with zero reserved bytes, this function searches for one ** with nReserve reserved bytes at the end of it. ** ** If successful, set variable p->detected_pgsz to the detected page-size ** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page ** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, ** if an error occurs (e.g. an IO or OOM error), then an SQLite error code ** is returned. The final value of p->detected_pgsz is undefined in this ** case. */ static int recoverVfsDetectPagesize( sqlite3_recover *p, /* Recover handle */ sqlite3_file *pFd, /* File-handle open on input database */ u32 nReserve, /* Possible nReserve value */ i64 nSz /* Size of database file in bytes */ ){ int rc = SQLITE_OK; const int nMin = 512; const int nMax = 65536; const int nMaxBlk = 4; u32 pgsz = 0; int iBlk = 0; u8 *aPg = 0; u8 *aTmp = 0; int nBlk = 0; aPg = (u8*)sqlite3_malloc(2*nMax); if( aPg==0 ) return SQLITE_NOMEM; aTmp = &aPg[nMax]; nBlk = (nSz+nMax-1)/nMax; if( nBlk>nMaxBlk ) nBlk = nMaxBlk; do { for(iBlk=0; rc==SQLITE_OK && iBlk<nBlk; iBlk++){ int nByte = (nSz>=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); memset(aPg, 0, nMax); rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); if( rc==SQLITE_OK ){ int pgsz2; for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ int iOff; for(iOff=0; iOff<nMax; iOff+=pgsz2){ if( recoverIsValidPage(aTmp, &aPg[iOff], pgsz2-nReserve) ){ pgsz = pgsz2; break; } } } } } if( pgsz>(u32)p->detected_pgsz ){ p->detected_pgsz = pgsz; p->nReserve = nReserve; } if( nReserve==0 ) break; nReserve = 0; }while( 1 ); p->detected_pgsz = pgsz; sqlite3_free(aPg); return rc; } /* ** The xRead() method of the wrapper VFS. This is used to intercept calls ** to read page 1 of the input database. */ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ int rc = SQLITE_OK; if( pFd->pMethods==&recover_methods ){ pFd->pMethods = recover_g.pMethods; rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); if( nByte==16 ){ sqlite3_randomness(16, aBuf); }else if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ /* Ensure that the database has a valid header file. The only fields ** that really matter to recovery are: ** ** + Database page size (16-bits at offset 16) ** + Size of db in pages (32-bits at offset 28) ** + Database encoding (32-bits at offset 56) ** ** Also preserved are: ** ** + first freelist page (32-bits at offset 32) ** + size of freelist (32-bits at offset 36) ** + the wal-mode flags (16-bits at offset 18) ** ** We also try to preserve the auto-vacuum, incr-value, user-version ** and application-id fields - all 32 bit quantities at offsets ** 52, 60, 64 and 68. All other fields are set to known good values. ** ** Byte offset 105 should also contain the page-size as a 16-bit ** integer. */ const int aPreserve[] = {32, 36, 52, 60, 64, 68}; u8 aHdr[108] = { 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2e, 0x5b, 0x30, 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 }; u8 *a = (u8*)aBuf; u32 pgsz = recoverGetU16(&a[16]); u32 nReserve = a[20]; u32 enc = recoverGetU32(&a[56]); u32 dbsz = 0; i64 dbFileSize = 0; int ii; sqlite3_recover *p = recover_g.p; if( pgsz==0x01 ) pgsz = 65536; rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); if( rc==SQLITE_OK && p->detected_pgsz==0 ){ rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); } if( p->detected_pgsz ){ pgsz = p->detected_pgsz; nReserve = p->nReserve; } if( pgsz ){ dbsz = dbFileSize / pgsz; } if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ enc = SQLITE_UTF8; } sqlite3_free(p->pPage1Cache); p->pPage1Cache = 0; p->pPage1Disk = 0; p->pgsz = nByte; p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); if( p->pPage1Cache ){ p->pPage1Disk = &p->pPage1Cache[nByte]; memcpy(p->pPage1Disk, aBuf, nByte); aHdr[18] = a[18]; aHdr[19] = a[19]; recoverPutU32(&aHdr[28], dbsz); recoverPutU32(&aHdr[56], enc); recoverPutU16(&aHdr[105], pgsz-nReserve); if( pgsz==65536 ) pgsz = 1; recoverPutU16(&aHdr[16], pgsz); aHdr[20] = nReserve; for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){ memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4); } memcpy(aBuf, aHdr, sizeof(aHdr)); memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr)); memcpy(p->pPage1Cache, aBuf, nByte); }else{ rc = p->errCode; } } pFd->pMethods = &recover_methods; }else{ rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); } return rc; } /* ** Used to make sqlite3_io_methods wrapper methods less verbose. */ #define RECOVER_VFS_WRAPPER … /* ** Methods of the wrapper VFS. All methods except for xRead() and xClose() ** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent ** method on the lower level VFS, then reinstall the wrapper before returning. ** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. */ static int recoverVfsWrite( sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff ){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) ); } static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xTruncate(pFd, size) ); } static int recoverVfsSync(sqlite3_file *pFd, int flags){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xSync(pFd, flags) ); } static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xFileSize(pFd, pSize) ); } static int recoverVfsLock(sqlite3_file *pFd, int eLock){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xLock(pFd, eLock) ); } static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xUnlock(pFd, eLock) ); } static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xCheckReservedLock(pFd, pResOut) ); } static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ RECOVER_VFS_WRAPPER ( (pFd->pMethods ? pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND) ); } static int recoverVfsSectorSize(sqlite3_file *pFd){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xSectorSize(pFd) ); } static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xDeviceCharacteristics(pFd) ); } static int recoverVfsShmMap( sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp ){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) ); } static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xShmLock(pFd, offset, n, flags) ); } static void recoverVfsShmBarrier(sqlite3_file *pFd){ if( pFd->pMethods==&recover_methods ){ pFd->pMethods = recover_g.pMethods; pFd->pMethods->xShmBarrier(pFd); pFd->pMethods = &recover_methods; }else{ pFd->pMethods->xShmBarrier(pFd); } } static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xShmUnmap(pFd, deleteFlag) ); } static int recoverVfsFetch( sqlite3_file *pFd, sqlite3_int64 iOff, int iAmt, void **pp ){ (void)pFd; (void)iOff; (void)iAmt; *pp = 0; return SQLITE_OK; } static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){ (void)pFd; (void)iOff; (void)p; return SQLITE_OK; } /* ** Install the VFS wrapper around the file-descriptor open on the input ** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held ** when this function is called. */ static void recoverInstallWrapper(sqlite3_recover *p){ sqlite3_file *pFd = 0; assert( recover_g.pMethods==0 ); recoverAssertMutexHeld(); sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); assert( pFd==0 || pFd->pMethods!=&recover_methods ); if( pFd && pFd->pMethods ){ int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0); recover_g.pMethods = pFd->pMethods; recover_g.p = p; recover_methods.iVersion = iVersion; pFd->pMethods = &recover_methods; } } /* ** Uninstall the VFS wrapper that was installed around the file-descriptor open ** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be ** held when this function is called. */ static void recoverUninstallWrapper(sqlite3_recover *p){ sqlite3_file *pFd = 0; recoverAssertMutexHeld(); sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); if( pFd && pFd->pMethods ){ pFd->pMethods = recover_g.pMethods; recover_g.pMethods = 0; recover_g.p = 0; } } /* ** This function does the work of a single sqlite3_recover_step() call. It ** is guaranteed that the handle is not in an error state when this ** function is called. */ static void recoverStep(sqlite3_recover *p){ assert( p && p->errCode==SQLITE_OK ); switch( p->eState ){ case RECOVER_STATE_INIT: /* This is the very first call to sqlite3_recover_step() on this object. */ recoverSqlCallback(p, "BEGIN"); recoverSqlCallback(p, "PRAGMA writable_schema = on"); recoverEnterMutex(); recoverInstallWrapper(p); /* Open the output database. And register required virtual tables and ** user functions with the new handle. */ recoverOpenOutput(p); /* Open transactions on both the input and output databases. */ sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0); recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); recoverExec(p, p->dbIn, "BEGIN"); if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); recoverTransferSettings(p); recoverOpenRecovery(p); recoverCacheSchema(p); recoverUninstallWrapper(p); recoverLeaveMutex(); recoverExec(p, p->dbOut, "BEGIN"); recoverWriteSchema1(p); p->eState = RECOVER_STATE_WRITING; break; case RECOVER_STATE_WRITING: { if( p->w1.pTbls==0 ){ recoverWriteDataInit(p); } if( SQLITE_DONE==recoverWriteDataStep(p) ){ recoverWriteDataCleanup(p); if( p->zLostAndFound ){ p->eState = RECOVER_STATE_LOSTANDFOUND1; }else{ p->eState = RECOVER_STATE_SCHEMA2; } } break; } case RECOVER_STATE_LOSTANDFOUND1: { if( p->laf.pUsed==0 ){ recoverLostAndFound1Init(p); } if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ p->eState = RECOVER_STATE_LOSTANDFOUND2; } break; } case RECOVER_STATE_LOSTANDFOUND2: { if( p->laf.pAllAndParent==0 ){ recoverLostAndFound2Init(p); } if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ p->eState = RECOVER_STATE_LOSTANDFOUND3; } break; } case RECOVER_STATE_LOSTANDFOUND3: { if( p->laf.pInsert==0 ){ recoverLostAndFound3Init(p); } if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ p->eState = RECOVER_STATE_SCHEMA2; } break; } case RECOVER_STATE_SCHEMA2: { int rc = SQLITE_OK; recoverWriteSchema2(p); p->eState = RECOVER_STATE_DONE; /* If no error has occurred, commit the write transaction on the output ** database. Regardless of whether or not an error has occurred, make ** an attempt to end the read transaction on the input database. */ recoverExec(p, p->dbOut, "COMMIT"); rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); if( p->errCode==SQLITE_OK ) p->errCode = rc; recoverSqlCallback(p, "PRAGMA writable_schema = off"); recoverSqlCallback(p, "COMMIT"); p->eState = RECOVER_STATE_DONE; recoverFinalCleanup(p); break; }; case RECOVER_STATE_DONE: { /* no-op */ break; }; } } /* ** This is a worker function that does the heavy lifting for both init ** functions: ** ** sqlite3_recover_init() ** sqlite3_recover_init_sql() ** ** All this function does is allocate space for the recover handle and ** take copies of the input parameters. All the real work is done within ** sqlite3_recover_run(). */ sqlite3_recover *recoverInit( sqlite3* db, const char *zDb, const char *zUri, /* Output URI for _recover_init() */ int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ void *pSqlCtx /* Context arg for _recover_init_sql() */ ){ sqlite3_recover *pRet = 0; int nDb = 0; int nUri = 0; int nByte = 0; if( zDb==0 ){ zDb = "main"; } nDb = recoverStrlen(zDb); nUri = recoverStrlen(zUri); nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; pRet = (sqlite3_recover*)sqlite3_malloc(nByte); if( pRet ){ memset(pRet, 0, nByte); pRet->dbIn = db; pRet->zDb = (char*)&pRet[1]; pRet->zUri = &pRet->zDb[nDb+1]; memcpy(pRet->zDb, zDb, nDb); if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri); pRet->xSql = xSql; pRet->pSqlCtx = pSqlCtx; pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; } return pRet; } /* ** Initialize a recovery handle that creates a new database containing ** the recovered data. */ sqlite3_recover *sqlite3_recover_init( sqlite3* db, const char *zDb, const char *zUri ){ return recoverInit(db, zDb, zUri, 0, 0); } /* ** Initialize a recovery handle that returns recovered data in the ** form of SQL statements via a callback. */ sqlite3_recover *sqlite3_recover_init_sql( sqlite3* db, const char *zDb, int (*xSql)(void*, const char*), void *pSqlCtx ){ return recoverInit(db, zDb, 0, xSql, pSqlCtx); } /* ** Return the handle error message, if any. */ const char *sqlite3_recover_errmsg(sqlite3_recover *p){ return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; } /* ** Return the handle error code. */ int sqlite3_recover_errcode(sqlite3_recover *p){ return p ? p->errCode : SQLITE_NOMEM; } /* ** Configure the handle. */ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ int rc = SQLITE_OK; if( p==0 ){ rc = SQLITE_NOMEM; }else if( p->eState!=RECOVER_STATE_INIT ){ rc = SQLITE_MISUSE; }else{ switch( op ){ case 789: /* This undocumented magic configuration option is used to set the ** name of the auxiliary database that is ATTACH-ed to the database ** connection and used to hold state information during the ** recovery process. This option is for debugging use only and ** is subject to change or removal at any time. */ sqlite3_free(p->zStateDb); p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); break; case SQLITE_RECOVER_LOST_AND_FOUND: { const char *zArg = (const char*)pArg; sqlite3_free(p->zLostAndFound); if( zArg ){ p->zLostAndFound = recoverMPrintf(p, "%s", zArg); }else{ p->zLostAndFound = 0; } break; } case SQLITE_RECOVER_FREELIST_CORRUPT: p->bFreelistCorrupt = *(int*)pArg; break; case SQLITE_RECOVER_ROWIDS: p->bRecoverRowid = *(int*)pArg; break; case SQLITE_RECOVER_SLOWINDEXES: p->bSlowIndexes = *(int*)pArg; break; default: rc = SQLITE_NOTFOUND; break; } } return rc; } /* ** Do a unit of work towards the recovery job. Return SQLITE_OK if ** no error has occurred but database recovery is not finished, SQLITE_DONE ** if database recovery has been successfully completed, or an SQLite ** error code if an error has occurred. */ int sqlite3_recover_step(sqlite3_recover *p){ if( p==0 ) return SQLITE_NOMEM; if( p->errCode==SQLITE_OK ) recoverStep(p); if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ return SQLITE_DONE; } return p->errCode; } /* ** Do the configured recovery operation. Return SQLITE_OK if successful, or ** else an SQLite error code. */ int sqlite3_recover_run(sqlite3_recover *p){ while( SQLITE_OK==sqlite3_recover_step(p) ); return sqlite3_recover_errcode(p); } /* ** Free all resources associated with the recover handle passed as the only ** argument. The results of using a handle with any sqlite3_recover_** ** API function after it has been passed to this function are undefined. ** ** A copy of the value returned by the first call made to sqlite3_recover_run() ** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has ** not been called on this handle. */ int sqlite3_recover_finish(sqlite3_recover *p){ int rc; if( p==0 ){ rc = SQLITE_NOMEM; }else{ recoverFinalCleanup(p); if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); if( p->errCode==SQLITE_OK ) p->errCode = rc; } rc = p->errCode; sqlite3_free(p->zErrMsg); sqlite3_free(p->zStateDb); sqlite3_free(p->zLostAndFound); sqlite3_free(p->pPage1Cache); sqlite3_free(p); } return rc; } #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /************************* End ../ext/recover/sqlite3recover.c ********************/ # endif /* SQLITE_HAVE_SQLITE3R */ #endif #ifdef SQLITE_SHELL_EXTSRC # include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) #endif #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session */ typedef struct OpenSession OpenSession; struct OpenSession { char *zName; /* Symbolic name for this session */ int nFilter; /* Number of xFilter rejection GLOB patterns */ char **azFilter; /* Array of xFilter rejection GLOB patterns */ sqlite3_session *p; /* The open session */ }; #endif ExpertInfo; struct ExpertInfo { … }; /* A single line in the EQP output */ EQPGraphRow; struct EQPGraphRow { … }; /* All EQP output is collected into an instance of the following */ EQPGraph; struct EQPGraph { … }; /* Parameters affecting columnar mode result display (defaulting together) */ ColModeOpts; #define ColModeOpts_default … #define ColModeOpts_default_qbox … /* ** State information about the database connection is contained in an ** instance of the following structure. */ ShellState; struct ShellState { … }; #ifdef SQLITE_SHELL_FIDDLE static ShellState shellState; #endif /* Allowed values for ShellState.autoEQP */ #define AUTOEQP_off … #define AUTOEQP_on … #define AUTOEQP_trigger … #define AUTOEQP_full … /* Allowed values for ShellState.openMode */ #define SHELL_OPEN_UNSPEC … #define SHELL_OPEN_NORMAL … #define SHELL_OPEN_APPENDVFS … #define SHELL_OPEN_ZIPFILE … #define SHELL_OPEN_READONLY … #define SHELL_OPEN_DESERIALIZE … #define SHELL_OPEN_HEXDB … /* Allowed values for ShellState.eTraceType */ #define SHELL_TRACE_PLAIN … #define SHELL_TRACE_EXPANDED … #define SHELL_TRACE_NORMALIZED … /* Bits in the ShellState.flgProgress variable */ #define SHELL_PROGRESS_QUIET … #define SHELL_PROGRESS_RESET … #define SHELL_PROGRESS_ONCE … /* ** These are the allowed shellFlgs values */ #define SHFLG_Pagecache … #define SHFLG_Lookaside … #define SHFLG_Backslash … #define SHFLG_PreserveRowid … #define SHFLG_Newlines … #define SHFLG_CountChanges … #define SHFLG_Echo … #define SHFLG_HeaderSet … #define SHFLG_DumpDataOnly … #define SHFLG_DumpNoSys … #define SHFLG_TestingMode … /* ** Macros for testing and setting shellFlgs */ #define ShellHasFlag(P,X) … #define ShellSetFlag(P,X) … #define ShellClearFlag(P,X) … /* ** These are the allowed modes. */ #define MODE_Line … #define MODE_Column … #define MODE_List … #define MODE_Semi … #define MODE_Html … #define MODE_Insert … #define MODE_Quote … #define MODE_Tcl … #define MODE_Csv … #define MODE_Explain … #define MODE_Ascii … #define MODE_Pretty … #define MODE_EQP … #define MODE_Json … #define MODE_Markdown … #define MODE_Table … #define MODE_Box … #define MODE_Count … #define MODE_Off … #define MODE_ScanExp … static const char *modeDescr[] = …; /* ** These are the column/row/line separators used by the various ** import/export modes. */ #define SEP_Column … #define SEP_Row … #define SEP_Tab … #define SEP_Space … #define SEP_Comma … #define SEP_CrLf … #define SEP_Unit … #define SEP_Record … /* ** Limit input nesting via .read or any other input redirect. ** It's not too expensive, so a generous allowance can be made. */ #define MAX_INPUT_NESTING … /* ** A callback for the sqlite3_log() interface. */ static void shellLog(void *pArg, int iErrCode, const char *zMsg){ … } /* ** SQL function: shell_putsnl(X) ** ** Write the text X to the screen (or whatever output is being directed) ** adding a newline at the end, and then return X. */ static void shellPutsFunc( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … } /* ** If in safe mode, print an error message described by the arguments ** and exit immediately. */ static void failIfSafeMode( ShellState *p, const char *zErrMsg, ... ){ … } /* ** SQL function: edit(VALUE) ** edit(VALUE,EDITOR) ** ** These steps: ** ** (1) Write VALUE into a temporary file. ** (2) Run program EDITOR on that temporary file. ** (3) Read the temporary file back and return its content as the result. ** (4) Delete the temporary file ** ** If the EDITOR argument is omitted, use the value in the VISUAL ** environment variable. If still there is no EDITOR, through an error. ** ** Also throw an error if the EDITOR program returns a non-zero exit code. */ #ifndef SQLITE_NOHAVE_SYSTEM static void editFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ … } #endif /* SQLITE_NOHAVE_SYSTEM */ /* ** Save or restore the current output mode */ static void outputModePush(ShellState *p){ … } static void outputModePop(ShellState *p){ … } /* ** Output the given string as a hex-encoded blob (eg. X'1234' ) */ static void output_hex_blob(const void *pBlob, int nBlob){ … } /* ** Find a string that is not found anywhere in z[]. Return a pointer ** to that string. ** ** Try to use zA and zB first. If both of those are already found in z[] ** then make up some string and store it in the buffer zBuf. */ static const char *unused_string( const char *z, /* Result must not appear anywhere in z */ const char *zA, const char *zB, /* Try these first */ char *zBuf /* Space to store a generated string */ ){ … } /* ** Output the given string as a quoted string using SQL quoting conventions. ** ** See also: output_quoted_escaped_string() */ static void output_quoted_string(const char *z){ … } /* ** Output the given string as a quoted string using SQL quoting conventions. ** Additionallly , escape the "\n" and "\r" characters so that they do not ** get corrupted by end-of-line translation facilities in some operating ** systems. ** ** This is like output_quoted_string() but with the addition of the \r\n ** escape mechanism. */ static void output_quoted_escaped_string(const char *z){ … } /* ** Find earliest of chars within s specified in zAny. ** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. */ static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ … } /* ** Output the given string as a quoted according to C or TCL quoting rules. */ static void output_c_string(const char *z){ … } /* ** Output the given string as a quoted according to JSON quoting rules. */ static void output_json_string(const char *z, i64 n){ … } /* ** Output the given string with characters that are special to ** HTML escaped. */ static void output_html_string(const char *z){ … } /* ** If a field contains any character identified by a 1 in the following ** array, then the string must be quoted for CSV. */ static const char needCsvQuote[] = …; /* ** Output a single term of CSV. Actually, p->colSeparator is used for ** the separator, which may or may not be a comma. p->nullValue is ** the null value. Strings are quoted if necessary. The separator ** is only issued if bSep is true. */ static void output_csv(ShellState *p, const char *z, int bSep){ … } /* ** This routine runs when the user presses Ctrl-C */ static void interrupt_handler(int NotUsed){ … } #if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) /* ** This routine runs for console events (e.g. Ctrl-C) on Win32 */ static BOOL WINAPI ConsoleCtrlHandler( DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ ){ if( dwCtrlType==CTRL_C_EVENT ){ interrupt_handler(0); return TRUE; } return FALSE; } #endif #ifndef SQLITE_OMIT_AUTHORIZATION /* ** This authorizer runs in safe mode. */ static int safeModeAuth( void *pClientData, int op, const char *zA1, const char *zA2, const char *zA3, const char *zA4 ){ … } /* ** When the ".auth ON" is set, the following authorizer callback is ** invoked. It always returns SQLITE_OK. */ static int shellAuth( void *pClientData, int op, const char *zA1, const char *zA2, const char *zA3, const char *zA4 ){ … } #endif /* ** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. ** ** If the schema statement in z[] contains a start-of-comment and if ** sqlite3_complete() returns false, try to terminate the comment before ** printing the result. https://sqlite.org/forum/forumpost/d7be961c5c */ static void printSchemaLine(const char *z, const char *zTail){ … } static void printSchemaLineN(char *z, int n, const char *zTail){ … } /* ** Return true if string z[] has nothing but whitespace and comments to the ** end of the first line. */ static int wsToEol(const char *z){ … } /* ** Add a new entry to the EXPLAIN QUERY PLAN data */ static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ … } /* ** Free and reset the EXPLAIN QUERY PLAN data that has been collected ** in p->sGraph. */ static void eqp_reset(ShellState *p){ … } /* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after ** pOld, or return the first such line if pOld is NULL */ static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ … } /* Render a single level of the graph that has iEqpId as its parent. Called ** recursively to render sublevels. */ static void eqp_render_level(ShellState *p, int iEqpId){ … } /* ** Display and reset the EXPLAIN QUERY PLAN data */ static void eqp_render(ShellState *p, i64 nCycle){ … } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* ** Progress handler callback. */ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ oputf("Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ oputf("Progress %u\n", p->nProgress); } return 0; } #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ /* ** Print N dashes */ static void print_dashes(int N){ … } /* ** Print a markdown or table-style row separator using ascii-art */ static void print_row_separator( ShellState *p, int nArg, const char *zSep ){ … } /* ** This is the callback routine that the shell ** invokes for each row of a query result. */ static int shell_callback( void *pArg, int nArg, /* Number of result columns */ char **azArg, /* Text of each result column */ char **azCol, /* Column names */ int *aiType /* Column types. Might be NULL */ ){ … } /* ** This is the callback routine that the SQLite library ** invokes for each row of a query result. */ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ … } /* ** This is the callback routine from sqlite3_exec() that appends all ** output onto the end of a ShellText object. */ static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ … } /* ** Generate an appropriate SELFTEST table in the main database. */ static void createSelftestTable(ShellState *p){ … } /* ** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the ** table name. */ static void set_table_name(ShellState *p, const char *zName){ … } /* ** Maybe construct two lines of text that point out the position of a ** syntax error. Return a pointer to the text, in memory obtained from ** sqlite3_malloc(). Or, if the most recent error does not involve a ** specific token that we can point to, return an empty string. ** ** In all cases, the memory returned is obtained from sqlite3_malloc64() ** and should be released by the caller invoking sqlite3_free(). */ static char *shell_error_context(const char *zSql, sqlite3 *db){ … } /* ** Execute a query statement that will generate SQL output. Print ** the result columns, comma-separated, on a line and then add a ** semicolon terminator to the end of that line. ** ** If the number of columns is 1 and that column contains text "--" ** then write the semicolon on a separate line. That way, if a ** "--" comment occurs at the end of the statement, the comment ** won't consume the semicolon terminator. */ static int run_table_dump_query( ShellState *p, /* Query context */ const char *zSelect /* SELECT statement to extract content */ ){ … } /* ** Allocate space and save off string indicating current error. */ static char *save_err_msg( sqlite3 *db, /* Database to query */ const char *zPhase, /* When the error occurs */ int rc, /* Error code returned from API */ const char *zSql /* SQL string, or NULL */ ){ … } #ifdef __linux__ /* ** Attempt to display I/O stats on Linux using /proc/PID/io */ static void displayLinuxIoStats(void){ … } #endif /* ** Display a single line of status using 64-bit values. */ static void displayStatLine( char *zLabel, /* Label for this one line */ char *zFormat, /* Format for the result */ int iStatusCtrl, /* Which status to display */ int bReset /* True to reset the stats */ ){ … } /* ** Display memory stats. */ static int display_stats( sqlite3 *db, /* Database to query */ ShellState *pArg, /* Pointer to ShellState */ int bReset /* True to reset the stats */ ){ … } #ifdef SQLITE_ENABLE_STMT_SCANSTATUS static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ int iPid = 0; int ret = 1; sqlite3_stmt_scanstatus_v2(p, iEntry, SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid ); while( iPid!=0 ){ int ii; for(ii=0; 1; ii++){ int iId; int res; res = sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId ); if( res ) break; if( iId==iPid ){ sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid ); } } ret++; } return ret; } #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ static const int f = SQLITE_SCANSTAT_COMPLEX; sqlite3_stmt *p = pArg->pStmt; int ii = 0; i64 nTotal = 0; int nWidth = 0; eqp_reset(pArg); for(ii=0; 1; ii++){ const char *z = 0; int n = 0; if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ break; } n = (int)strlen(z) + scanStatsHeight(p, ii)*3; if( n>nWidth ) nWidth = n; } nWidth += 4; sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); for(ii=0; 1; ii++){ i64 nLoop = 0; i64 nRow = 0; i64 nCycle = 0; int iId = 0; int iPid = 0; const char *zo = 0; const char *zName = 0; char *zText = 0; double rEst = 0.0; if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ break; } sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); zText = sqlite3_mprintf("%s", zo); if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ char *z = 0; if( nCycle>=0 && nTotal>0 ){ z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, nCycle, ((nCycle*100)+nTotal/2) / nTotal ); } if( nLoop>=0 ){ z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); } if( nRow>=0 ){ z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); } if( zName && pArg->scanstatsOn>1 ){ double rpl = (double)nRow / (double)nLoop; z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); } zText = sqlite3_mprintf( "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z ); } eqp_append(pArg, iId, iPid, zText); sqlite3_free(zText); } eqp_render(pArg, nTotal); } #endif /* ** Parameter azArray points to a zero-terminated array of strings. zStr ** points to a single nul-terminated string. Return non-zero if zStr ** is equal, according to strcmp(), to any of the strings in the array. ** Otherwise, return zero. */ static int str_in_array(const char *zStr, const char **azArray){ … } /* ** If compiled statement pSql appears to be an EXPLAIN statement, allocate ** and populate the ShellState.aiIndent[] array with the number of ** spaces each opcode should be indented before it is output. ** ** The indenting rules are: ** ** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent ** all opcodes that occur between the p2 jump destination and the opcode ** itself by 2 spaces. ** ** * Do the previous for "Return" instructions for when P2 is positive. ** See tag-20220407a in wherecode.c and vdbe.c. ** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: ** Yield SeekGt SeekLt RowSetRead Rewind ** or if the P1 parameter is one instead of zero, ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ … }static void explain_data_delete(ShellState *p){ … }static void exec_prepared_stmt(ShellState*, sqlite3_stmt*)static void display_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ … }static unsigned int savedSelectTracestatic unsigned int savedWhereTracestatic void disable_debug_trace_modes(void){ … }static void restore_debug_trace_modes(void){ … }static void bind_table_init(ShellState *p){ … }static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ … }#define BOX_24 …#define BOX_13 …#define BOX_23 …#define BOX_34 …#define BOX_12 …#define BOX_14 …#define BOX_123 …#define BOX_134 …#define BOX_234 …#define BOX_124 …#define BOX_1234 …static void print_box_line(int N){ … }static void print_box_row_separator( ShellState *p, int nArg, const char *zSep1, const char *zSep2, const char *zSep3 ){ … }static char *translateForDisplayAndDup( const unsigned char *z, /* Input text to be transformed */ const unsigned char **pzTail, /* OUT: Tail of the input for next line */ int mxWidth, /* Max width. 0 means no limit */ u8 bWordWrap /* If true, avoid breaking mid-word */ ){ … }static char *quoted_column(sqlite3_stmt *pStmt, int i){ … }static void exec_prepared_stmt_columnar( ShellState *p, /* Pointer to ShellState */ sqlite3_stmt *pStmt /* Statement to run */ ){ … }static void exec_prepared_stmt( ShellState *pArg, /* Pointer to ShellState */ sqlite3_stmt *pStmt /* Statement to run */ ){ … }#ifndef SQLITE_OMIT_VIRTUALTABLEstatic int expertHandleSQL( ShellState *pState, const char *zSql, char **pzErr ){ … }static int expertFinish( ShellState *pState, int bCancel, char **pzErr ){ … }static int expertDotCommand( ShellState *pState, /* Current shell tool state */ char **azArg, /* Array of arguments passed to dot command */ int nArg /* Number of entries in azArg[] */ ){ … }#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */static int shell_exec( ShellState *pArg, /* Pointer to ShellState */ const char *zSql, /* SQL to be evaluated */ char **pzErrMsg /* Error msg written here */ ){ … }static void freeColumnList(char **azCol){ … }static char **tableColumnList(ShellState *p, const char *zTab){ … }static void toggleSelectOrder(sqlite3 *db){ … }static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ … }static int run_schema_dump_query( ShellState *p, const char *zQuery ){ … }static const char *(azHelp[]) = …static int showHelp(FILE *out, const char *zPattern){ … }static int process_input(ShellState *p)static char *readFile(const char *zName, int *pnByte){ … }#if defined(SQLITE_ENABLE_SESSION)#endif#if defined(SQLITE_ENABLE_SESSION)#else#define session_close_all(X,Y) …#endif#if defined(SQLITE_ENABLE_SESSION)#endifint deduceDatabaseType(const char *zName, int dfltZip){ … }#ifndef SQLITE_OMIT_DESERIALIZEstatic unsigned char *readHexDb(ShellState *p, int *pnData){ … }#endif /* SQLITE_OMIT_DESERIALIZE */static void shellUSleepFunc( sqlite3_context *context, int argcUnused, sqlite3_value **argv ){ … }#define OPEN_DB_KEEPALIVE …#define OPEN_DB_ZIPFILE …static void open_db(ShellState *p, int openFlags){ … }void close_db(sqlite3 *db){ … }#if HAVE_READLINE || HAVE_EDITLINE#elif HAVE_LINENOISE#endifstatic void resolve_backslashes(char *z){ … }static int booleanValue(const char *zArg){ … }static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ … }static void output_file_close(FILE *f){ … }static FILE *output_file_open(const char *zFile, int bTextMode){ … }#ifndef SQLITE_OMIT_TRACEstatic int sql_trace_callback( unsigned mType, /* The trace type */ void *pArg, /* The ShellState pointer */ void *pP, /* Usually a pointer to sqlite_stmt */ void *pX /* Auxiliary output */ ){ … }#endifstatic void test_breakpoint(void){ … }ImportCtxstruct ImportCtx { … }static void import_cleanup(ImportCtx *p){ … }static void import_append_char(ImportCtx *p, int c){ … }static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ … }static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ … }static void tryToCloneData( ShellState *p, sqlite3 *newDb, const char *zTable ){ … }static void tryToCloneSchema( ShellState *p, sqlite3 *newDb, const char *zWhere, void (*xForEach)(ShellState*,sqlite3*,const char*) ){ … }static void tryToClone(ShellState *p, const char *zNewDb){ … }#ifndef SQLITE_SHELL_FIDDLEstatic void output_redir(ShellState *p, FILE *pfNew){ … }static void output_reset(ShellState *p){ … }#else#define output_redir …#define output_reset …#endifstatic int db_int(sqlite3 *db, const char *zSql){ … }#if SQLITE_SHELL_HAVE_RECOVERstatic unsigned int get2byteInt(unsigned char *a){ … }static unsigned int get4byteInt(unsigned char *a){ … }static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ … }#endif /* SQLITE_SHELL_HAVE_RECOVER */static int shellDatabaseError(sqlite3 *db){ … }static int testcase_glob(const char *zGlob, const char *z){ … }static int optionMatch(const char *zStr, const char *zOpt){ … }int shellDeleteFile(const char *zFilename){ … }static void clearTempFile(ShellState *p){ … }static void newTempFile(ShellState *p, const char *zSuffix){ … }static void shellFkeyCollateClause( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ … }static int lintFkeyIndexes( ShellState *pState, /* Current shell tool state */ char **azArg, /* Array of arguments passed to dot command */ int nArg /* Number of entries in azArg[] */ ){ … }static int lintDotCommand( ShellState *pState, /* Current shell tool state */ char **azArg, /* Array of arguments passed to dot command */ int nArg /* Number of entries in azArg[] */ ){ … }static void shellPrepare( sqlite3 *db, int *pRc, const char *zSql, sqlite3_stmt **ppStmt ){ … }static void shellPreparePrintf( sqlite3 *db, int *pRc, sqlite3_stmt **ppStmt, const char *zFmt, ... ){ … }static void shellFinalize( int *pRc, sqlite3_stmt *pStmt ){ … }#if !defined SQLITE_OMIT_VIRTUALTABLEvoid shellReset( int *pRc, sqlite3_stmt *pStmt ){ … }#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)#define AR_CMD_CREATE …#define AR_CMD_UPDATE …#define AR_CMD_INSERT …#define AR_CMD_EXTRACT …#define AR_CMD_LIST …#define AR_CMD_HELP …#define AR_CMD_REMOVE …#define AR_SWITCH_VERBOSE …#define AR_SWITCH_FILE …#define AR_SWITCH_DIRECTORY …#define AR_SWITCH_APPEND …#define AR_SWITCH_DRYRUN …#define AR_SWITCH_GLOB …#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */#if SQLITE_SHELL_HAVE_RECOVERstatic int recoverSqlCb(void *pCtx, const char *zSql){ … }static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ … }#endif /* SQLITE_SHELL_HAVE_RECOVER */static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ … }#ifdef SHELL_DEBUG#define rc_err_oom_die …#elsestatic void rc_err_oom_die(int rc){ … }#endif#ifdef SHELL_COLFIX_DB /* If this is set, the DB can be in a file. */#else /* Otherwise, memory is faster/better for the transient DB. */static const char *zCOL_DB = …#endif#ifndef SHELL_AUTOCOLUMN_SEP#define AUTOCOLUMN_SEP …#else#define AUTOCOLUMN_SEP …#endifstatic char *zAutoColumn(const char *zColNew, sqlite3 **pDb, char **pzRenamed){ … }static int outputDumpWarning(ShellState *p, const char *zLike){ … }static struct { … } faultsim_state = …static int faultsim_callback(int iArg){ … }static int do_meta_command(char *zLine, ShellState *p){ … }#ifndef CHAR_BIT#define CHAR_BIT …#endifQuickScanState#define QSS_SETV(qss, newst) …#define QSS_INPLAIN(qss) …#define QSS_PLAINWHITE(qss) …#define QSS_PLAINDARK(qss) …#define QSS_SEMITERM(qss) …static QuickScanState quickscan(char *zLine, QuickScanState qss, SCAN_TRACKER_REFTYPE pst){ … }static int line_is_command_terminator(char *zLine){ … }#ifdef SQLITE_OMIT_COMPLETE# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE.#endifstatic int line_is_complete(char *zSql, int nSql){ … }static int doAutoDetectRestore(ShellState *p, const char *zSql){ … }static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ … }static void echo_group_input(ShellState *p, const char *zDo){ … }#ifdef SQLITE_SHELL_FIDDLE#endif /* SQLITE_SHELL_FIDDLE */static int process_input(ShellState *p){ … }static char *find_home_dir(int clearFlag){ … }static const char *find_xdg_config(void){ … }static void process_sqliterc( ShellState *p, /* Configuration data */ const char *sqliterc_override /* Name of config file. NULL to use default */ ){ … }static const char zOptions[] = …#ifdef SQLITE_ENABLE_VFSTRACE#endif#ifdef SQLITE_HAVE_ZLIB#endifstatic void usage(int showDetail){ … }static void verify_uninitialized(void){ … }static void main_init(ShellState *data) { … }#if defined(_WIN32) || defined(WIN32)#if !SQLITE_OS_WINRT#endif#if !SQLITE_OS_WINRT#endif#elsestatic void printBold(const char *zText){ … }#endifstatic char *cmdline_option_value(int argc, char **argv, int i){ … }static void sayAbnormalExit(void){ … }#ifndef SQLITE_SHELL_IS_UTF8# if (defined(_WIN32) || defined(WIN32)) \ && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__)))#define SQLITE_SHELL_IS_UTF8 …# else#define SQLITE_SHELL_IS_UTF8 …# endif#endif#ifdef SQLITE_SHELL_FIDDLE#define main …#endif#if SQLITE_SHELL_IS_UTF8int SQLITE_CDECL main(int argc, char **argv){ … }#ifdef SQLITE_SHELL_FIDDLE#endif /* SQLITE_SHELL_FIDDLE */
Codebase Browser
chromium