From 4ffcf620b0cb3aa9c593b78f1d8e3579c47c507c Mon Sep 17 00:00:00 2001 From: easydatawarehousing Date: Mon, 28 Jul 2014 13:05:41 +0200 Subject: [PATCH] Added Oracle source databases --- CMakeLists.txt | 1 + Documentation/eto-features-futures.html | 1 + Documentation/eto-index.html | 1 + Documentation/eto-requirements.html | 2 +- ETO_Common/LICENSE INIPARSER | 25 + ETO_Common/src/eto_util.c | 27 +- EasyMTOra/src/eto_dbs.c | 8 +- EasyNTOra/CMakeLists.txt | 35 + EasyNTOra/INSTALL.md | 1 + EasyNTOra/Oracle_testfile.ini | 37 + EasyNTOra/README.md | 1 + EasyNTOra/src/eto_dbs.c | 921 ++ EasySTOra/include/sqlite3.h | 175 +- EasySTOra/src/sqlite3.c | 10560 ++++++++++------ EasyTTOra/INSTALL.md | 2 +- INSTALL.md | 2 +- PLSQL/install_utl_eto.sql | 55 +- README.md | 2 +- VERSION | 6 + msvc/Easy-To-Oracle.sln | 28 + .../This is where dll and sym files go | 0 msvc/EasyNTOra/EasyNTOra.vcxproj | 154 + msvc/EasyNTOra/EasyNTOra.vcxproj.filters | 63 + msvc/EasyNTOra/EasyNTOra.vcxproj.user | 7 + .../Release/This is where dll files go | 0 msvc/EasyPTOra/EasyPTOra.vcxproj | 3 + 26 files changed, 8058 insertions(+), 4059 deletions(-) create mode 100644 ETO_Common/LICENSE INIPARSER create mode 100644 EasyNTOra/CMakeLists.txt create mode 100644 EasyNTOra/INSTALL.md create mode 100644 EasyNTOra/Oracle_testfile.ini create mode 100644 EasyNTOra/README.md create mode 100644 EasyNTOra/src/eto_dbs.c create mode 100644 msvc/EasyNTOra/Debug/This is where dll and sym files go create mode 100644 msvc/EasyNTOra/EasyNTOra.vcxproj create mode 100644 msvc/EasyNTOra/EasyNTOra.vcxproj.filters create mode 100644 msvc/EasyNTOra/EasyNTOra.vcxproj.user create mode 100644 msvc/EasyNTOra/Release/This is where dll files go diff --git a/CMakeLists.txt b/CMakeLists.txt index f4103e2..9f9511b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,3 +18,4 @@ add_subdirectory(EasyMTOra) add_subdirectory(EasyPTOra) add_subdirectory(EasySTOra) add_subdirectory(EasyTTOra) +add_subdirectory(EasyNTOra) diff --git a/Documentation/eto-features-futures.html b/Documentation/eto-features-futures.html index 8a2869d..727c0d5 100644 --- a/Documentation/eto-features-futures.html +++ b/Documentation/eto-features-futures.html @@ -56,6 +56,7 @@ • SQLite 3.x
• Microsoft Access 97 - 2003 (discontinued in ETO 2.0 version)
• Presto (Hadoop)
+• Oracle

• All datatypes are handled except: (var)binary, lob's, image. Fields with these datatypes are automatically filtered out by PL/SQL package.
Other fieldtypes are converted to corresponding Oracle types. Mostly to varchar2, number and date. User- defined conversions are possible, as long as they are valid for external tables. Character fields with content longer than 4,000 bytes are cut of at that length (extended varchar length of Oracle 12c is partially supported). Character set conversions are carried out by Oracle.
diff --git a/Documentation/eto-index.html b/Documentation/eto-index.html index 60e5b39..65bbaf1 100644 --- a/Documentation/eto-index.html +++ b/Documentation/eto-index.html @@ -54,6 +54,7 @@ • All functions can be used from within the database using SQL or PL/SQL.
• Simple to install and use
• Can be used for any kind of data-integration, like loading of a datawarehouse, data migration and even creating a data-virtualization platform
+• Binary releases (i.e. precompiled executables) are available via: easydatawarehousing.com

More information
• Learn how it works. See the Technical overview, the comparison with other data-integration methods and the Technical Requirements
diff --git a/Documentation/eto-requirements.html b/Documentation/eto-requirements.html index d80259f..eaf74fa 100644 --- a/Documentation/eto-requirements.html +++ b/Documentation/eto-requirements.html @@ -47,7 +47,7 @@ Easy ... To Oracle - Requirements
Oracle
-ETO uses the Oracle external table "pre-processor" feature. This feature is available from Oracle releases 10.2.0.5, 11.1.0.7 and 11.2.0.1.
+ETO uses the Oracle external table "pre-processor" feature. This feature is available from Oracle releases 10.2.0.5, 11.1.0.7, 11.2.0.1 and 12.*.
Any edition of the database will do (Express, Standard, Enterprise). Since the "pre-processor" feature was developed for RAC it should also work in a RAC environment, but this has not been tested by me.

OS
diff --git a/ETO_Common/LICENSE INIPARSER b/ETO_Common/LICENSE INIPARSER new file mode 100644 index 0000000..d7f4465 --- /dev/null +++ b/ETO_Common/LICENSE INIPARSER @@ -0,0 +1,25 @@ +Iniparser + +Copyright (c) 2000-2012 by Nicolas Devillard. +MIT License + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/ETO_Common/src/eto_util.c b/ETO_Common/src/eto_util.c index 0d49f57..c50ae33 100644 --- a/ETO_Common/src/eto_util.c +++ b/ETO_Common/src/eto_util.c @@ -34,7 +34,8 @@ extern int __cdecl tolower(_In_ int _C); ETO_PARAMS* eto_new_ini_params(const char* ini_file_name) { ETO_PARAMS* params; - unsigned int i; + unsigned int i, j; + bool copy_port = true; if (!ini_file_name) { @@ -81,10 +82,26 @@ ETO_PARAMS* eto_new_ini_params(const char* ini_file_name) // Check if tcp port is included in database string for (i = 0; i < strlen(params->dbs); i++) { + // Look for a colon if (strncmp(params->dbs + i, ":", 1) == 0) { - params->prt = atoi(params->dbs + i + 1); - params->dbs[i] = 0; // Modifying an iniparser string here ! + // Check that the portnumber is not followed by a string + // for instance an Oracle servicename + for (j = i + 1; j < strlen(params->dbs); j++) + { + if ( ! (strncmp(params->dbs + j, "0", 1) >= 0 && strncmp(params->dbs + j, "9", 1) <= 0) ) + { + copy_port = false; + break; + } + } + + if (copy_port) + { + params->prt = atoi(params->dbs + i + 1); + params->dbs[i] = 0; // Modifying an iniparser string here ! + } + break; } } @@ -300,7 +317,9 @@ bool eto_create_query(char **sql, const char *in_statement, const char *in_table tempsql = (char*)calloc(strlen(*sql) + 1, sizeof(char) ); if (!tempsql) - exit(1); strcpy(tempsql, *sql); + exit(1); + + strcpy(tempsql, *sql); strcpy(*sql, "SELECT * FROM ("); strcat(*sql, tempsql); strcat(*sql, ") WHERE 1=0"); diff --git a/EasyMTOra/src/eto_dbs.c b/EasyMTOra/src/eto_dbs.c index 5fe5bf6..c06df73 100644 --- a/EasyMTOra/src/eto_dbs.c +++ b/EasyMTOra/src/eto_dbs.c @@ -272,7 +272,7 @@ bool eto_open_database(ETO_PARAMS* params) } // Check/copy/create sql - if (!eto_create_query(&qdata->sql, params->sql, params->tbl, false /* is_describe: always false for freetdp */, true) ) + if (!eto_create_query(&qdata->sql, params->sql, params->tbl, false /* is_describe: always false for MySQL */, true) ) { // Log messages added by eto_create_query return false; @@ -283,20 +283,20 @@ bool eto_open_database(ETO_PARAMS* params) if (!conn) { - eto_log_add_entry_s("ETO-10202 Open database: Connection allocation failed. MySQL client version: ", mysql_get_client_info() ); + eto_log_add_entry_s("ETO-10202 Open database: Connection allocation failed. MySQL client version:", mysql_get_client_info() ); return false; } // Login to database if (mysql_real_connect(conn, params->dbs, params->usr, params->pwd, NULL, params->prt, NULL, 0) == NULL) { - eto_log_add_entry_s("ETO-10206 Open database: Create connection failed: ", mysql_error(conn)); + eto_log_add_entry_s("ETO-10206 Open database: Create connection failed:", mysql_error(conn)); return false; } if (mysql_set_character_set(conn, "utf8")) { - eto_log_add_entry_s("ETO-10207 Open database: Could not set UTF-8 character set. Default is: ", mysql_character_set_name(conn) ); + eto_log_add_entry_s("ETO-10207 Open database: Could not set UTF-8 character set. Default is:", mysql_character_set_name(conn) ); } // Execute query diff --git a/EasyNTOra/CMakeLists.txt b/EasyNTOra/CMakeLists.txt new file mode 100644 index 0000000..7c89ce4 --- /dev/null +++ b/EasyNTOra/CMakeLists.txt @@ -0,0 +1,35 @@ +# Set target +set(TARGET_NAME "EasyNTOra") + +# Add libraries +set(ORACLE_OCI_NAMES libclntsh.so) +set(ORACLE_LIB_DIR "${CMAKE_SOURCE_DIR}/bin/lib") + +find_library(ORACLE_OCI_LIBRARY + NAMES ${ORACLE_OCI_NAMES} + PATHS ${ORACLE_LIB_DIR} + NO_DEFAULT_PATH +) + +set(LIBS ${ORACLE_OCI_LIBRARY}) + +# Add include dirs +include_directories ("${CMAKE_SOURCE_DIR}/ETO_Common/include") +include_directories ("${CMAKE_SOURCE_DIR}/EasyNTOra/instantclient/include") + +# Add sources +set(SRC + src/eto_dbs.c + ${CMAKE_SOURCE_DIR}/ETO_Common/src/dictionary.c + ${CMAKE_SOURCE_DIR}/ETO_Common/src/iniparser.c + ${CMAKE_SOURCE_DIR}/ETO_Common/src/eto_log.c + ${CMAKE_SOURCE_DIR}/ETO_Common/src/eto_util.c + ${CMAKE_SOURCE_DIR}/ETO_Common/src/eto_main.c +) + +# Add executable and dependancies. Setup RPATH to the executable (lib) directory +add_executable (${TARGET_NAME} ${SRC}) + +SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,'$ORIGIN:$ORIGIN/lib'" ) + +target_link_libraries (${TARGET_NAME} ${LIBS}) \ No newline at end of file diff --git a/EasyNTOra/INSTALL.md b/EasyNTOra/INSTALL.md new file mode 100644 index 0000000..78c2e9e --- /dev/null +++ b/EasyNTOra/INSTALL.md @@ -0,0 +1 @@ +Easy ... To Oracle - ORACLE ========================== #### Executable name: EasyNTOra #### Used library: ORACLE Instant Client / OCI #### Minimum library version: 11.1.0.6.0 #### Recommended library version: 12.1.0.1.0 Preparations ------------ Download the Oracle Instant Client from: http://www.oracle.com/technetwork/database/features/instant-client/index-097480.html Grab these 2 packages for your platform: Instant Client Package - Basic Lite Instant Client Package - SDK Extract these somewhere. Copy these files: Windows Headerfiles: from instantclient/sdk/include to EasyNTOra/instantclient/include .lib, files: instantclient to EasyNTOra/instantclient/lib (only need oci.lib) .dll/.sym files: instantclient to msvc/EasyNTOra/Debug (only: oci.dll, oraons.dll, oraociicus12.dll, plus the corresponding .sym files) .dll files: instantclient to msvc/EasyNTOra/Release (only: oci.dll, oraons.dll, oraociicus12.dll) Linux/Debian Headerfiles: from instantclient/sdk/include to EasyNTOra/instantclient/include .so files: instantclient to bin/lib (libclntsh.so.12.1 libclntshcore.so.12.1 libnnz12.so libociicus.so lobons.so ) Create a symbolic link (libclntsh.so => libclntsh.so.12.1) in order to work with cmake 2.6: cd bin/lib ln -s libclntsh.so.12.1 libclntsh.so Building EasyNTO ---------------- See the general installation instructions. Specific usage information -------------------------- None Test script ----------- DECLARE r number; BEGIN DBMS_OUTPUT.enable(20000); r := SYS.UTL_ETO.ETO_Create_Database_Connection('ORACLE'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_executable, 'EasyNTOra'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_bin_dir, 'user_dir_eto_bin'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_data_dir, 'user_dir_eto_data'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_db_name, 'ipadress_or_servername:port/servicename'); -- ORACLE uses 1521 as default tcp port, the port and servicename are both optional r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_db_user, 'user'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_db_pwd, 'password'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_default_charset, 'AL32UTF8'); r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_def_cacherows, '25'); -- For Oracle sources a cache size between 15-45 works fastest -- r := SYS.UTL_ETO.ETO_Set_Parameter('ORACLE', SYS.UTL_ETO.Eto_par_list_table_query, 'select owner || ''.'' || table_name from all_tables where owner=''schemaname'''); -- Replace schemaname with the actual name !!! -- r := SYS.UTL_ETO.ETO_Create_Tables_From_DB ( pOutputSchema => 'SCHEMA' -- replace , pOutputPrefix => 'ORA_' , pConnectionname => 'ORACLE' , pDropAllowed => 'Y' ); DBMS_OUTPUT.PUT_LINE('Return value = ' || r); END; ------------------------------------- Copyright (C) 2011-2014 Ivo Herweijer \ No newline at end of file diff --git a/EasyNTOra/Oracle_testfile.ini b/EasyNTOra/Oracle_testfile.ini new file mode 100644 index 0000000..3162ca3 --- /dev/null +++ b/EasyNTOra/Oracle_testfile.ini @@ -0,0 +1,37 @@ +# This file is part of Easy to Oracle - Free Open Source Data Integration + +# Copyright (C) 2011-2014 Ivo Herweijer + +# Easy to Oracle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can contact me via email: info@easydatawarehousing.com +# +# This is an example ini file +# Note that entries starting with an underscore _ +# are not read. These were added as examples. +# + +[EasyXTOra] +database = server/servicename; +user = user; +password = pwd; +_table = schema.tablename; +sql = select * from dual; +cache_rows = 25; +logfilename = Oracle_testfile.log; +_describe_id = 123; # If used, ETO will return field information, not data +fieldseparator = ";"; # Default value is \2 Valid values are: \t , ; +lineseparator = \n; # Default value is \1 Valid values are: \r \n \r\n +addheader = 1; +maxvarcharlen = 4000; diff --git a/EasyNTOra/README.md b/EasyNTOra/README.md new file mode 100644 index 0000000..c76cc0f --- /dev/null +++ b/EasyNTOra/README.md @@ -0,0 +1 @@ +Easy to Oracle - Free open source data integration ================================================== Oracle To Oracle, why ? ----------------------- Datareplication from an Oracle source database to an Oracle target database! Why I hear you ask. Isn't a database link or datapump easier and faster ? This is absolutely true. Database links are easier to install (part of the database) and easier to use. And is in fact about two times faster than easy-2-oracle. So why add this option? Well, this part of easy-2-oracle wasn't meant for Oracle target databases, but for versions ported to other target databases/platforms, like R. EasyNTOra --------- Name of the executable should be EasyOTOra, but this name is already in use for 'Office' databases. I used the nearest letter in the alphabet instead. \ No newline at end of file diff --git a/EasyNTOra/src/eto_dbs.c b/EasyNTOra/src/eto_dbs.c new file mode 100644 index 0000000..2cdb059 --- /dev/null +++ b/EasyNTOra/src/eto_dbs.c @@ -0,0 +1,921 @@ +/* This file is part of Easy to Oracle - Free Open Source Data Integration +* +* Copyright (C) 2011-2014 Ivo Herweijer +* +* Easy to Oracle is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +* +* You can contact me via email: info@easydatawarehousing.com +*/ + +#include "eto_dbs.h" +#include "eto_log.h" +#include +#include +#include + +#define MAX_FIELDTYPE_LENGTH 30 +#define DATETIMEFIELD_LENGTH 40 +#define UTF8_LANGUAGECODE 873 + +// Describe info +typedef struct ST_ETO_ORA_COLUMNINFO +{ + char *fieldname; + ub2 oracletype; + char *fieldtype; + ub2 size; + ub1 precision; + sb1 scale; + ub1 notnull; + ub4 value_buffer_size; +} ETO_ORA_COLUMNINFO; + +static ETO_ORA_COLUMNINFO* eto_new_columninfo() +{ + ETO_ORA_COLUMNINFO* column = (ETO_ORA_COLUMNINFO*)malloc( sizeof(ETO_ORA_COLUMNINFO) ); + if (!column) + exit(1); + + column->fieldname = NULL; + column->oracletype = 0; + column->fieldtype = NULL; + column->size = 0; + column->precision = 0; + column->scale = 0; + column->notnull = 0; + column->value_buffer_size = 0; + + return column; +} + +static void eto_delete_columninfo(ETO_ORA_COLUMNINFO* column) +{ + if (!column) + return; + + if (column->fieldname) + free(column->fieldname); + + if (column->fieldtype) + free(column->fieldtype); + + free(column); +} + +static bool eto_continue(OCIError *error, sword status, char* errormessage) +{ + text message[512]; + sb4 errcode = 0; + + // Only check when we actually want to report something + if (!errormessage) + return true; + + // See page: OCI Programming Basics 2-21 + switch (status) + { + case OCI_SUCCESS_WITH_INFO: + OCIErrorGet( (dvoid *)error, (ub4)1, (text *)NULL, &errcode, message, (ub4)sizeof(message), OCI_HTYPE_ERROR); + eto_log_add_entry_s(errormessage, message); + return true; + + case OCI_NEED_DATA: + eto_log_add_entry_s(errormessage, "no runtime data available"); + return false; + + case OCI_NO_DATA: + eto_log_add_entry_s(errormessage, "no (further) data available"); + return false; + + case OCI_ERROR: + OCIErrorGet( (dvoid *)error, (ub4)1, (text *)NULL, &errcode, message, (ub4)sizeof(message), OCI_HTYPE_ERROR); + eto_log_add_entry_s(errormessage, message); + return false; + + case OCI_INVALID_HANDLE: + eto_log_add_entry_s(errormessage, "Invalid handle used"); + return false; + + case OCI_STILL_EXECUTING: + eto_log_add_entry_s(errormessage, "nonblocking mode, current operation could not be completed immediately"); + return true; + + case OCI_CONTINUE: + eto_log_add_entry_s(errormessage, "Callback completed"); + return true; + + default: + return true; + } + + return false; +} + +typedef struct ST_ETO_ORA_CONNECTION +{ + // Oracle handles + OCIEnv *environment; + OCIError *error; + OCISvcCtx *servercontext; + OCIStmt *stmt; + + // Statement results + ub4 numcolumns; + ETO_ORA_COLUMNINFO **columninfo; + char **value; + OCIDefine **defs; + sb2 **indicator; + ub2 **valuelength; +} ETO_ORA_CONNECTION; + +static ETO_ORA_CONNECTION* eto_new_connection() +{ + ETO_ORA_CONNECTION* conn = (ETO_ORA_CONNECTION*)malloc( sizeof(ETO_ORA_CONNECTION) ); + if (!conn) + exit(1); + + conn->environment = NULL; + conn->error = NULL; + conn->servercontext = NULL; + conn->stmt = NULL; + conn->numcolumns = 0; + conn->columninfo = NULL; + conn->value = NULL; + conn->defs = NULL; + conn->indicator = NULL; + conn->valuelength = NULL; + + return conn; +} + +static void eto_delete_connection(ETO_ORA_CONNECTION* conn) +{ + ub4 i; + + if (!conn) + return; + + // Delete arrays + if (conn->numcolumns && conn->columninfo) + { + for (i = 0; i < conn->numcolumns; i++) + { + eto_delete_columninfo(conn->columninfo[i]); + + if (conn->value && conn->value[i]) + free(conn->value[i]); + + //free(conn->defs[i]); + + if (conn->indicator && conn->indicator[i]) + free(conn->indicator[i]); + + if (conn->valuelength && conn->valuelength[i]) + free(conn->valuelength[i]); + } + + free(conn->columninfo); + + if (conn->value) + free(conn->value); + + //free(conn->defs); + + if(conn->indicator) + free(conn->indicator); + + if(conn->valuelength) + free(conn->valuelength); + } + + // Logout + if (conn->servercontext) + { + eto_continue(conn->error, OCILogoff(conn->servercontext, conn->error), NULL); + } + + // Free all handles + if (conn->environment) + { + OCIHandleFree( (dvoid *) conn->stmt, OCI_HTYPE_STMT); + OCIHandleFree( (dvoid *) conn->servercontext, OCI_HTYPE_SVCCTX); + OCIHandleFree( (dvoid *) conn->error, OCI_HTYPE_ERROR); + OCIHandleFree( (dvoid *) conn->environment, OCI_HTYPE_ENV); + } + + free(conn); +} + +static OraText* eto_oracle_connectstring(char* dbname, int port) +{ + int i, colonpos, slashpos, servernamelength; + bool copy_port = true; + OraText *connectstring; + + // Create connect string (like: host:1521/XE) from databasename and/or port + // Note that a servicename is optional + i = strlen(dbname) + 6 /* port=5 + terminator=1 */; + connectstring = (OraText*)calloc(i, sizeof(OraText) ); + if (!connectstring) + exit(1); + memset(connectstring, 0, i); + + // Port given ? + if (port < 1 || port > 65535) + { + // No port specified => use databasename as is (may contain a port/servicename) + strcpy(connectstring, dbname); + return connectstring; + } + + // Modify connectstring to add port + + // Check if tcp port and/or servicename is included in the database string + colonpos = -1; + slashpos = -1; + + for (i = 0; i < (int)strlen(dbname); i++) + { + if (strncmp(dbname + i, ":", 1) == 0) + { + colonpos = i; + break; + } + } + + for (i = colonpos > 0 ? colonpos + 1 : 0; i < (int)strlen(dbname); i++) + { + if (strncmp(dbname + i, "/", 1) == 0) + { + slashpos = i; + break; + } + } + + // Copy servername + if (colonpos > 0) + servernamelength = colonpos; + else if (slashpos > 0) + servernamelength = slashpos; + else + servernamelength = strlen(dbname); + + // Create connectstring + sprintf(connectstring + , "%.*s:%d%s" + , servernamelength, dbname + , port + , slashpos > 0 ? dbname + slashpos : "" + ); + + return connectstring; +} + +static bool eto_oracle_connect(ETO_ORA_CONNECTION* conn, ETO_PARAMS* params) +{ + ub2 languagecode; + OraText languagename[65]; + OraText *connectstring = NULL; + OraText *setdateformatsql = "ALTER SESSION SET NLS_DATE_FORMAT='yyyy-mm-dd hh24:mi:ss' NLS_TIMESTAMP_FORMAT='yyyy-mm-dd hh24:mi:ss.ff'"; + + // Create environment + sword errorcode = OCIEnvNlsCreate( + (OCIEnv **)&conn->environment, + (ub4)OCI_DEFAULT, + (dvoid *)0, + (dvoid * (*)(dvoid *,size_t) ) 0, + (dvoid * (*)(dvoid *, dvoid *, size_t) ) 0, + (void (*)(dvoid *, dvoid *) ) 0, + (size_t)0, + (dvoid **) 0, + UTF8_LANGUAGECODE, + UTF8_LANGUAGECODE + ); + + if (errorcode != 0) + { + eto_log_add_entry_i("ETO-10202 Open database: Create Oracle environment failed with errorcode =", errorcode); + return false; + } + + // Create error handle + OCIHandleAlloc( (dvoid *) conn->environment, (dvoid **) &conn->error, OCI_HTYPE_ERROR, (size_t) 0, (dvoid **) 0); + + // Check characterset + languagecode = OCINlsCharSetNameToId( conn->environment, (text*)"AL32UTF8" ); + if (languagecode != UTF8_LANGUAGECODE) + { + // Add a warning to the log + OCINlsGetInfo(conn->environment, conn->error, languagename, 64, OCI_NLS_CHARACTER_SET); + eto_log_add_entry_s("ETO-10202 Open database: Warning: characterset could not be set to AL32UTF8, using:", languagename); + } + + // Construct connectstring + connectstring = eto_oracle_connectstring(params->dbs, params->prt); + + // Connect + if (!eto_continue( + conn->error, + OCILogon2( + conn->environment, + conn->error, + &conn->servercontext, + (text*)params->usr, + (ub4)strlen(params->usr), + (text*)params->pwd, + (ub4)strlen(params->pwd), + (text*)connectstring, + (ub4)strlen(connectstring), + OCI_DEFAULT + ), + "ETO-10206 Open database: Connect failed:") ) + { + eto_log_add_entry_s("ETO-10206 Open database: Tried to connect to:", connectstring); + free(connectstring); + return false; + } + + free(connectstring); + + // Init statement handle + if (!eto_continue( + conn->error, + OCIHandleAlloc( (dvoid *)conn->environment, (dvoid **)&conn->stmt, OCI_HTYPE_STMT, (size_t)0, (dvoid **)0), + "ETO-10207 Open database: prepare statement handle failed:") ) + return false; + + // Set date format + if (!eto_continue( + conn->error, + OCIStmtPrepare(conn->stmt, conn->error, setdateformatsql, (ub4)strlen(setdateformatsql), (ub4)OCI_NTV_SYNTAX, (ub4)OCI_DEFAULT), + "ETO-10224 Execute query: prepare dateformat query failed:") ) + return false; + + if(!eto_continue( + conn->error, + OCIStmtExecute(conn->servercontext, conn->stmt, conn->error, 1, 0, (OCISnapshot *)0, (OCISnapshot *)0, OCI_DEFAULT), + "ETO-10225 Execute query: query dateformat execution failed:") ) + return false; + + return true; +} + +static void eto_write_output(QUERYDATA *qdata, ETO_ORA_CONNECTION* conn) +{ + unsigned int i, newdatalen, lseplen, curr_cache_len = 0; + ub4 r, fetched_rows; + sword status; + bool continue_fetch = true; + sb4 error_code = 0; + text error_message[10]; + + if (qdata->is_describe || !conn->numcolumns) + return; + + lseplen = strlen(qdata->lsep); + + while(true) + { + if( (status = OCIStmtFetch2(conn->stmt, conn->error, qdata->cache_rows, OCI_DEFAULT, 0, OCI_DEFAULT) ) != OCI_SUCCESS) + { + OCIErrorGet( (dvoid *)conn->error, (ub4)1, (text *)NULL, &error_code, error_message, 10, OCI_HTYPE_ERROR); + + // Check if this is the last fetch (with less rows than requested) + if (status == OCI_NO_DATA && error_code == 1403 /* No Data Found */) + continue_fetch = false; + else + { + // Log the error and quit + eto_continue(conn->error, status, "ETO-10243 Queryresults: Fetch next record failed:"); + break; + } + } + + // Get actual number of rows fetched + if (!eto_continue( + conn->error, + OCIAttrGet((void *)conn->stmt, OCI_HTYPE_STMT, (void *)&fetched_rows, (ub4 *)0, OCI_ATTR_ROWS_FETCHED, conn->error), + "ETO-10245 Queryresults: get row count failed:") ) + return; + + // Process rows + for(r = 0; r < fetched_rows; r++) + { + for(i = 0; i < conn->numcolumns; i++) + { + // Get value length + newdatalen = conn->valuelength[i][r]; + if (!newdatalen) + newdatalen = 1; // Just a separator + + // Cut off at 4000 characters (shouldn't happen because maxlength was passed to oracle when binding) + if (newdatalen > qdata->vml) + newdatalen = qdata->vml; + + // Check cache size + if (qdata->cache_size < curr_cache_len + newdatalen + 3) + { + qdata->cache_size = ( (curr_cache_len + newdatalen + 1027) / 1024) * 1024; // add block(s) of 1 Kb + qdata->cache = (char*)realloc( (char*)qdata->cache, qdata->cache_size); + if (!qdata->cache) + exit(1); + } + + // Add value to cache if it isn't NULL + // See: Indicator variables: Oracle Call Interface Programmer's Guide, page 2-24 + if (conn->indicator[i][r] != -1) + strncat(qdata->cache, conn->value[i] + r * (conn->columninfo[i]->size + 1), newdatalen); + + curr_cache_len += newdatalen; + + if (i < conn->numcolumns - 1) + { + strcat(qdata->cache, qdata->fsep); + curr_cache_len++; + } + } + + strcat(qdata->cache, qdata->lsep); + curr_cache_len += lseplen; + + } + + // Print rows to output + printf("%s", qdata->cache); + qdata->cache[0] = 0; + curr_cache_len = 0; + + // Quit when last iteration found + if (!continue_fetch) + break; + } +} + +static bool eto_oracle_is_datetime(ub2 oracle_type) +{ + if (oracle_type == OCI_TYPECODE_DATE || + oracle_type == OCI_TYPECODE_TIME || + oracle_type == OCI_TYPECODE_TIME_TZ || + oracle_type == OCI_TYPECODE_TIMESTAMP || + oracle_type == OCI_TYPECODE_TIMESTAMP_TZ || + oracle_type == OCI_TYPECODE_TIMESTAMP_LTZ || + oracle_type == OCI_TYPECODE_INTERVAL_YM || + oracle_type == OCI_TYPECODE_INTERVAL_DS ) + return true; + + return false; +} + +static bool eto_describe_output(QUERYDATA *qdata, ETO_ORA_CONNECTION* conn) +{ + ub4 i, fieldname_length; + ub2 type; + OCIParam *column = NULL; + text *pfieldname; + ub2 col_width; + + // Get the number of columns in the query + if (!eto_continue( + conn->error, + OCIAttrGet((void *)conn->stmt, OCI_HTYPE_STMT, (void *)&conn->numcolumns, (ub4 *)0, OCI_ATTR_PARAM_COUNT, conn->error), + "ETO-10240 Queryresults: get column count failed:") ) + return false; + + if (conn->numcolumns == 0) + return false; + + + // Allocate columninfo + conn->columninfo = (ETO_ORA_COLUMNINFO**)calloc(conn->numcolumns, sizeof(ETO_ORA_COLUMNINFO*) ); + if (!conn->columninfo) + return false; + + for (i = 0; i < conn->numcolumns; i++) + conn->columninfo[i] = NULL; + + + // Copy header record to output + if (qdata->is_describe) + printf("%s", "describe_id;name;datatype;datatype_descr;maxlength;scale;precision;notnull;statement\n"); + + + // Get describe info for each column + for (i = 0; i < conn->numcolumns; i++) + { + // Allocate columninfo + conn->columninfo[i] = eto_new_columninfo(); + + + // Get column handle. See: Oracle Call Interface Programmer's Guide, page 6-12 + if (!eto_continue( + conn->error, + OCIParamGet( (void *)conn->stmt, OCI_HTYPE_STMT, conn->error, (void **)&column, i+1 /* plus one ! */), + "ETO-10240 Queryresults: Error getting columnhandle:" + ) ) + return false; + + // Get fieldtype + type = 0; + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&conn->columninfo[i]->oracletype, (ub4 *)0, OCI_ATTR_DATA_TYPE, conn->error), + "ETO-10240 Queryresults: Error getting column type:") ) + return false; + + // Get fieldname + fieldname_length = 0; + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void**)&pfieldname, (ub4 *)&fieldname_length, OCI_ATTR_NAME, conn->error), + "ETO-10240 Queryresults: Error getting column name:") ) + return false; + + conn->columninfo[i]->fieldname = (char*)calloc(fieldname_length + 1, sizeof(char) ); + if (!conn->columninfo[i]->fieldname) + exit(1); + memset(conn->columninfo[i]->fieldname, 0, fieldname_length + 1); + strncpy(conn->columninfo[i]->fieldname, pfieldname, fieldname_length); + + // Get fieldlength + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&col_width, (ub4 *)0, OCI_ATTR_CHAR_SIZE, conn->error), + "ETO-10240 Queryresults: Error getting field length:") ) + return false; + + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&conn->columninfo[i]->size, (ub4 *)0, OCI_ATTR_DATA_SIZE, conn->error), + "ETO-10240 Queryresults: Error getting column length:") ) + return false; + + // Adjust length for date/time types + if (eto_oracle_is_datetime(conn->columninfo[i]->oracletype) ) + conn->columninfo[i]->size = DATETIMEFIELD_LENGTH; + + // Get precision + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&conn->columninfo[i]->precision, (ub4 *)0, OCI_ATTR_PRECISION, conn->error), + "ETO-10240 Queryresults: Error getting column precision:") ) + return false; + + // Get scale + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&conn->columninfo[i]->scale, (ub4 *)0, OCI_ATTR_SCALE, conn->error), + "ETO-10240 Queryresults: Error getting column scale:") ) + return false; + + // Get not null + if (!eto_continue( + conn->error, + OCIAttrGet( (void *)column, OCI_DTYPE_PARAM, (void *)&conn->columninfo[i]->notnull, (ub4 *)0, OCI_ATTR_IS_NULL, conn->error), + "ETO-10240 Queryresults: Error getting column nullable:") ) + return false; + + // Allocate fieldtype + conn->columninfo[i]->fieldtype = (char*)calloc(MAX_FIELDTYPE_LENGTH + 1, sizeof(char) ); + if (!conn->columninfo[i]->fieldtype) + exit(1); + memset(conn->columninfo[i]->fieldtype, 0, MAX_FIELDTYPE_LENGTH + 1); + + // Describe column + switch (type) + { + case OCI_TYPECODE_REF: + case OCI_TYPECODE_PTR: + case OCI_TYPECODE_MLSLABEL: + case OCI_TYPECODE_VARRAY: + case OCI_TYPECODE_TABLE: + case OCI_TYPECODE_OBJECT: + case OCI_TYPECODE_OPAQUE: + case OCI_TYPECODE_NAMEDCOLLECTION: + case OCI_TYPECODE_BLOB: + case OCI_TYPECODE_BFILE: + case OCI_TYPECODE_CLOB: + case OCI_TYPECODE_CFILE: + case OCI_TYPECODE_OTMFIRST: + case OCI_TYPECODE_OTMLAST: + // case OCI_TYPECODE_SYSFIRST: + case OCI_TYPECODE_SYSLAST: + case OCI_TYPECODE_NCLOB: + case OCI_TYPECODE_RAW: + strcpy(conn->columninfo[i]->fieldtype, "ORACLE_RAW"); + type = OCI_TYPECODE_RAW; + break; + + case OCI_TYPECODE_SIGNED8: + case OCI_TYPECODE_SIGNED16: + case OCI_TYPECODE_SIGNED32: + case OCI_TYPECODE_REAL: + case OCI_TYPECODE_DOUBLE: + case OCI_TYPECODE_BFLOAT: + case OCI_TYPECODE_BDOUBLE: + case OCI_TYPECODE_FLOAT: + case OCI_TYPECODE_NUMBER: + case OCI_TYPECODE_DECIMAL: + case OCI_TYPECODE_UNSIGNED8: + case OCI_TYPECODE_UNSIGNED16: + case OCI_TYPECODE_UNSIGNED32: + case OCI_TYPECODE_OCTET: + case OCI_TYPECODE_SMALLINT: + case OCI_TYPECODE_INTEGER: + case OCI_TYPECODE_PLS_INTEGER: + strcpy(conn->columninfo[i]->fieldtype, "ORACLE_NUMBER"); + type = OCI_TYPECODE_NUMBER; + break; + + case OCI_TYPECODE_UROWID: + case OCI_TYPECODE_VARCHAR2: + case OCI_TYPECODE_CHAR: + case OCI_TYPECODE_VARCHAR: + case OCI_TYPECODE_NCHAR: + case OCI_TYPECODE_NVARCHAR2: + case OCI_TYPECODE_NONE: + { + strcpy(conn->columninfo[i]->fieldtype, "ORACLE_VARCHAR2"); + type = OCI_TYPECODE_VARCHAR2; + + // Take the largest of width/size. For strings this should give the size in bytes (not chars) + if (col_width > conn->columninfo[i]->size) + conn->columninfo[i]->size = col_width; + + break; + } + + case OCI_TYPECODE_DATE: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_DATE"); break; + case OCI_TYPECODE_TIME: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_TIME"); break; + case OCI_TYPECODE_TIME_TZ: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_TIME_TZ"); break; + case OCI_TYPECODE_TIMESTAMP: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_TIMESTAMP"); break; + case OCI_TYPECODE_TIMESTAMP_TZ: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_TIMESTAMP_TZ"); break; + case OCI_TYPECODE_TIMESTAMP_LTZ: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_TIMESTAMP_LTZ"); break; + case OCI_TYPECODE_INTERVAL_YM: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_INTERVAL_YM"); break; + case OCI_TYPECODE_INTERVAL_DS: strcpy(conn->columninfo[i]->fieldtype, "ORACLE_INTERVAL_DS"); break; + + default: + strcpy(conn->columninfo[i]->fieldtype, "ORACLE_VARCHAR2"); + type = OCI_TYPECODE_VARCHAR2; + break; + } + + // Adjust fieldsize to fit maximum allowed width + if (conn->columninfo[i]->size > qdata->vml) + conn->columninfo[i]->size = qdata->vml; + + if (qdata->is_describe) + { + // Copy description to output + printf("%d%s%s%s%d%s%s%s%d%s%d%s%d%s%s%s%s%s" + , qdata->describe_id + , qdata->fsep + , conn->columninfo[i]->fieldname + , qdata->fsep + , conn->columninfo[i]->oracletype + 70000 + , qdata->fsep + , conn->columninfo[i]->fieldtype + , qdata->fsep + , conn->columninfo[i]->size + , qdata->fsep + , conn->columninfo[i]->scale + , qdata->fsep + , conn->columninfo[i]->precision + , qdata->fsep + , conn->columninfo[i]->notnull ? "Y" : "N" + , qdata->fsep + , qdata->sql + , qdata->lsep + ); + } + else + { + // Copy fieldnames to output + if (qdata->hdr && !qdata->hdr_printed) + printf("%s%s", i > 0 ? qdata->fsep : "", conn->columninfo[i]->fieldname); + } + } + + if (!qdata->is_describe && qdata->hdr) + printf("%s", qdata->lsep); + + qdata->hdr_printed = true; + + return true; +} + +static bool eto_bind_output(QUERYDATA *qdata, ETO_ORA_CONNECTION *conn) +{ + ub4 i; + + // Allocate define buffer + conn->defs = (OCIDefine**)calloc(conn->numcolumns, sizeof(OCIDefine*) ); + if (!conn->defs) + exit(1); + + // Allocate value buffer: per column there is one char buffer that holds all field data + // for 'qdata->cache_rows' rows. + // Skipsize is thus: conn->columninfo[i]->size+1 + // See: Binding and Defining in OCI page 5-17 + conn->value = (char**)calloc(conn->numcolumns, sizeof(char*) ); + if (!conn->value) + exit(1); + + for (i = 0; i < conn->numcolumns; i++) + { + conn->columninfo[i]->value_buffer_size = qdata->cache_rows * (conn->columninfo[i]->size+1) * sizeof(char); + + conn->value[i] = (char*)malloc(conn->columninfo[i]->value_buffer_size); + if (!conn->value[i]) + exit(1); + + memset(conn->value[i], 0, conn->columninfo[i]->value_buffer_size); + } + + // Allocate indicator buffer: 2D array of sb2. Skipsize is: sizeof(sb2) + conn->indicator = (sb2**)calloc(conn->numcolumns, sizeof(sb2*) ); + if (!conn->indicator) + exit(1); + + for (i = 0; i < conn->numcolumns; i++) + { + conn->indicator[i] = (sb2*)calloc(qdata->cache_rows, sizeof(sb2) ); + if (!conn->indicator[i]) + exit(1); + } + + // Allocate value-length buffer: 2D array of ub2. Skipsize is: sizeof(ub2) + conn->valuelength = (ub2**)calloc(conn->numcolumns, sizeof(ub2*) ); + if (!conn->valuelength) + exit(1); + + for (i = 0; i < conn->numcolumns; i++) + { + conn->valuelength[i] = (ub2*)calloc(qdata->cache_rows, sizeof(ub2) ); + if (!conn->valuelength[i]) + exit(1); + } + + // Bind all columns + for (i = 0; i < conn->numcolumns; i++) + { + conn->defs[i] = NULL; + + // Define (=bind) outputdata + if (!eto_continue( + conn->error, + OCIDefineByPos( + conn->stmt, + &conn->defs[i], + conn->error, + (ub4)i+1 /* plus one ! */, + (void *)conn->value[i], // GCC Warning typeconverion + (sb4)conn->columninfo[i]->size, + SQLT_STR, + (void *)conn->indicator[i], + (ub2 *)conn->valuelength[i], + (ub2 *)NULL, + OCI_DEFAULT + ), + "ETO-10242 Queryresults: Error binding column:" + ) ) + return false; + + // Define output arrays + if (!eto_continue( + conn->error, + OCIDefineArrayOfStruct( + conn->defs[i], + conn->error, + conn->columninfo[i]->size+1, // Skipsize for value buffer + sizeof(sb2), // Skipsize for indicator buffer + sizeof(ub2), // Skipsize for value-length buffer + 0 + ), + "ETO-10242 Queryresults: Error binding column:" + ) ) + return false; + + } + + return true; +} + +static bool eto_execute_query(QUERYDATA *qdata, ETO_ORA_CONNECTION *conn) +{ + int i = 0; + ub2 type = 0; + + // Prepare statement + if (!eto_continue( + conn->error, + OCIStmtPrepare(conn->stmt, conn->error, (OraText *)qdata->sql, (ub4)strlen(qdata->sql), (ub4)OCI_NTV_SYNTAX, (ub4)OCI_DEFAULT), + "ETO-10221 Execute query: prepare query failed:") ) + return false; + + // set the execution mode to OCI_DESCRIBE_ONLY to describe the request + if(!eto_continue( + conn->error, + OCIStmtExecute(conn->servercontext, conn->stmt, conn->error, 0, 0, (OCISnapshot *)0, (OCISnapshot *)0, OCI_DESCRIBE_ONLY), + "ETO-10222 Execute query: descripe step failed:") ) + return false; + + // Print headers/describe + if (!eto_describe_output(qdata, conn) ) + return false; + + // Execute statement again to get query results + if (!qdata->is_describe) + { + if (!eto_bind_output(qdata, conn) ) + return false; + + if(!eto_continue( + conn->error, + OCIStmtExecute(conn->servercontext, conn->stmt, conn->error, 0 /* Don't fetch any records yet */, 0, (OCISnapshot *)0, (OCISnapshot *)0, OCI_DEFAULT), + "ETO-10223 Execute query: query execution failed:") ) + return false; + + // Print rows + eto_write_output(qdata, conn); + } + + return true; +} + +bool eto_open_database(ETO_PARAMS* params) +{ + QUERYDATA *qdata; + ETO_ORA_CONNECTION *conn = eto_new_connection(); + + if (!conn) + return false; + + // Check parameters + if (!params || !params->dbs) + { + eto_log_add_entry("ETO-10110 No database servername or address given"); + return false; + } + + if (!params->tbl && !params->sql) + { + eto_log_add_entry("ETO-10113 No tablename or query given"); + return false; + } + + // Set up data + qdata = eto_new_querydata(params->sql, params->tbl); + + if (!qdata) + { + // Log messages added by eto_new_querydata + return false; + } + + qdata->is_describe = params->did > 0; + qdata->describe_id = params->did; + qdata->hdr = (bool)params->hdr || qdata->is_describe; + qdata->cache_rows = params->rws > 0 ? params->rws : 1; + qdata->vml = params->vml; + + // Convert separator strings + if (!eto_convert_separators(qdata->fsep, qdata->lsep, params->fsep, params->lsep) ) + { + // Log messages added by eto_convert_separators + return false; + } + + // Check/copy/create sql + if (!eto_create_query(&qdata->sql, params->sql, params->tbl, false /* is_describe: always false for Oracle */, false /* No show statements allowed */) ) + { + // Log messages added by eto_create_query + return false; + } + + // Connect to Oracle + if (!eto_oracle_connect(conn, params) ) + { + eto_log_add_entry("ETO-10202 Open database: Connect/login failed"); + return false; + } + + // Execute query + eto_execute_query(qdata, conn); + + // Cleanup + eto_delete_connection(conn); + eto_delete_querydata(qdata); + + return true; +} \ No newline at end of file diff --git a/EasySTOra/include/sqlite3.h b/EasySTOra/include/sqlite3.h index 6cf5977..0850726 100644 --- a/EasySTOra/include/sqlite3.h +++ b/EasySTOra/include/sqlite3.h @@ -107,9 +107,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.3.1" -#define SQLITE_VERSION_NUMBER 3008003 -#define SQLITE_SOURCE_ID "2014-02-11 14:52:19 ea3317a4803d71d88183b29f1d3086f46d68a00e" +#define SQLITE_VERSION "3.8.6" +#define SQLITE_VERSION_NUMBER 3008006 +#define SQLITE_SOURCE_ID "2014-07-24 23:23:26 114dcf33670fd98a1ebbac0e44f66b2d8bcccddf" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -560,7 +560,10 @@ SQLITE_API int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -575,6 +578,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels @@ -943,6 +947,12 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** +**
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -966,6 +976,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_HAS_MOVED 20 #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 /* ** CAPI3REF: Mutex Handle @@ -2026,9 +2037,13 @@ SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** ** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] ** is returned immediately upon encountering the lock. ^If the busy callback @@ -2037,16 +2052,18 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** or [SQLITE_IOERR_BLOCKED] to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -2078,10 +2095,12 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -2106,6 +2125,8 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** [database connection] any any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); @@ -2779,6 +2800,30 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** ** ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -2808,8 +2853,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -** file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +** file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. ** file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. ** @@ -4670,6 +4716,13 @@ SQLITE_API int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4688,6 +4741,11 @@ SQLITE_API int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. ** ** Note to Windows Runtime users: The temporary directory must be set ** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various @@ -6122,7 +6180,9 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 -#define SQLITE_TESTCTRL_LAST 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status @@ -7106,6 +7166,9 @@ SQLITE_API void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -7122,6 +7185,10 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** empty string, then a checkpoint is run on all databases of ** connection D. ^If the database connection D is not in ** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -7144,10 +7211,12 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish. Sync the db file if all frames in the log ** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (calls the busy-handler callback) until there is no +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database ** snapshot. It then checkpoints all frames in the log file and syncs the ** database file. This call blocks database writers while it is running, @@ -7155,7 +7224,8 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** **
    SQLITE_CHECKPOINT_RESTART
    ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, @@ -7345,6 +7415,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -7355,11 +7435,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, -#ifdef SQLITE_RTREE_INT_ONLY - int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), -#else - int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), -#endif + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -7371,11 +7447,60 @@ SQLITE_API int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #ifdef __cplusplus } /* end of the 'extern "C"' block */ diff --git a/EasySTOra/src/sqlite3.c b/EasySTOra/src/sqlite3.c index d0057a6..c72d13f 100644 --- a/EasySTOra/src/sqlite3.c +++ b/EasySTOra/src/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.8.3.1. By combining all the individual C code files into this +** version 3.8.6. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -25,6 +25,93 @@ #ifndef SQLITE_API # define SQLITE_API #endif +/************** Begin file sqliteInt.h ***************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Internal interface definitions for SQLite. +** +*/ +#ifndef _SQLITEINT_H_ +#define _SQLITEINT_H_ + +/* +** These #defines should enable >2GB file support on POSIX if the +** underlying operating system supports it. If the OS lacks +** large file support, or if the OS is windows, these should be no-ops. +** +** Ticket #2739: The _LARGEFILE_SOURCE macro must appear before any +** system #includes. Hence, this block of code must be the very first +** code in all source files. +** +** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch +** on the compiler command line. This is necessary if you are compiling +** on a recent machine (ex: Red Hat 7.2) but you want your code to work +** on an older machine (ex: Red Hat 6.0). If you compile on Red Hat 7.2 +** without this option, LFS is enable. But LFS does not exist in the kernel +** in Red Hat 6.0, so the code won't work. Hence, for maximum binary +** portability you should omit LFS. +** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. +** +** Similar is true for Mac OS X. LFS is only supported on Mac OS X 9 and later. +*/ +#ifndef SQLITE_DISABLE_LFS +# define _LARGE_FILE 1 +# ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +# endif +# define _LARGEFILE_SOURCE 1 +#endif + +/* +** For MinGW, check to see if we can include the header file containing its +** version information, among other things. Normally, this internal MinGW +** header file would [only] be included automatically by other MinGW header +** files; however, the contained version information is now required by this +** header file to work around binary compatibility issues (see below) and +** this is the only known way to reliably obtain it. This entire #if block +** would be completely unnecessary if there was any other way of detecting +** MinGW via their preprocessor (e.g. if they customized their GCC to define +** some MinGW-specific macros). When compiling for MinGW, either the +** _HAVE_MINGW_H or _HAVE__MINGW_H (note the extra underscore) macro must be +** defined; otherwise, detection of conditions specific to MinGW will be +** disabled. +*/ +#if defined(_HAVE_MINGW_H) +# include "mingw.h" +#elif defined(_HAVE__MINGW_H) +# include "_mingw.h" +#endif + +/* +** For MinGW version 4.x (and higher), check to see if the _USE_32BIT_TIME_T +** define is required to maintain binary compatibility with the MSVC runtime +** library in use (e.g. for Windows XP). +*/ +#if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ + defined(_WIN32) && !defined(_WIN64) && \ + defined(__MINGW_MAJOR_VERSION) && __MINGW_MAJOR_VERSION >= 4 && \ + defined(__MSVCRT__) +# define _USE_32BIT_TIME_T +#endif + +/* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear +** first in QNX. Also, the _USE_32BIT_TIME_T macro must appear first for +** MinGW. +*/ +/************** Include sqlite3.h in the middle of sqliteInt.h ***************/ /************** Begin file sqlite3.h *****************************************/ /* ** 2001 September 15 @@ -135,9 +222,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.8.3.1" -#define SQLITE_VERSION_NUMBER 3008003 -#define SQLITE_SOURCE_ID "2014-02-11 14:52:19 ea3317a4803d71d88183b29f1d3086f46d68a00e" +#define SQLITE_VERSION "3.8.6" +#define SQLITE_VERSION_NUMBER 3008006 +#define SQLITE_SOURCE_ID "2014-07-24 23:23:26 114dcf33670fd98a1ebbac0e44f66b2d8bcccddf" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -588,7 +675,10 @@ SQLITE_API int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. +** flag indicate that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -603,6 +693,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 /* ** CAPI3REF: File Locking Levels @@ -971,6 +1062,12 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** +**
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -994,6 +1091,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_HAS_MOVED 20 #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 /* ** CAPI3REF: Mutex Handle @@ -2054,9 +2152,13 @@ SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** ** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] ** is returned immediately upon encountering the lock. ^If the busy callback @@ -2065,16 +2167,18 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** or [SQLITE_IOERR_BLOCKED] to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -2106,10 +2210,12 @@ SQLITE_API int sqlite3_complete16(const void *sql); ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -2134,6 +2240,8 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** [database connection] any any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); @@ -2807,6 +2915,30 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
  • psow: ^The psow parameter may be "true" (or "on" or "yes" or +** "1") or "false" (or "off" or "no" or "0") to indicate that the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. ^The psow query +** parameter only works for the built-in unix and Windows VFSes. +** +**
  • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
  • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** ** ** ** ^Specifying an unknown parameter in the query component of a URI is not an @@ -2836,8 +2968,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** Open file "data.db" in the current directory for read-only access. ** Regardless of whether or not shared-cache mode is enabled by ** default, use a private cache. -** file:/home/fred/data.db?vfs=unix-nolock -** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +** file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. ** file:data.db?mode=readonly ** An error. "readonly" is not a valid option for the "mode" parameter. ** @@ -4698,6 +4831,13 @@ SQLITE_API int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4716,6 +4856,11 @@ SQLITE_API int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. ** ** Note to Windows Runtime users: The temporary directory must be set ** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various @@ -6150,7 +6295,9 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 -#define SQLITE_TESTCTRL_LAST 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status @@ -7134,6 +7281,9 @@ SQLITE_API void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -7150,6 +7300,10 @@ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** empty string, then a checkpoint is run on all databases of ** connection D. ^If the database connection D is not in ** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -7172,10 +7326,12 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish. Sync the db file if all frames in the log ** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (calls the busy-handler callback) until there is no +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database ** snapshot. It then checkpoints all frames in the log file and syncs the ** database file. This call blocks database writers while it is running, @@ -7183,7 +7339,8 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** **
    SQLITE_CHECKPOINT_RESTART
    ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, @@ -7373,6 +7530,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -7383,11 +7550,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, -#ifdef SQLITE_RTREE_INT_ONLY - int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), -#else - int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), -#endif + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -7399,11 +7562,60 @@ SQLITE_API int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #if 0 } /* end of the 'extern "C"' block */ @@ -7413,50 +7625,7 @@ struct sqlite3_rtree_geometry { /************** End of sqlite3.h *********************************************/ -/************** Begin file sqliteInt.h ***************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Internal interface definitions for SQLite. -** -*/ -#ifndef _SQLITEINT_H_ -#define _SQLITEINT_H_ - -/* -** These #defines should enable >2GB file support on POSIX if the -** underlying operating system supports it. If the OS lacks -** large file support, or if the OS is windows, these should be no-ops. -** -** Ticket #2739: The _LARGEFILE_SOURCE macro must appear before any -** system #includes. Hence, this block of code must be the very first -** code in all source files. -** -** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch -** on the compiler command line. This is necessary if you are compiling -** on a recent machine (ex: Red Hat 7.2) but you want your code to work -** on an older machine (ex: Red Hat 6.0). If you compile on Red Hat 7.2 -** without this option, LFS is enable. But LFS does not exist in the kernel -** in Red Hat 6.0, so the code won't work. Hence, for maximum binary -** portability you should omit LFS. -** -** Similar is true for Mac OS X. LFS is only supported on Mac OS X 9 and later. -*/ -#ifndef SQLITE_DISABLE_LFS -# define _LARGE_FILE 1 -# ifndef _FILE_OFFSET_BITS -# define _FILE_OFFSET_BITS 64 -# endif -# define _LARGEFILE_SOURCE 1 -#endif +/************** Continuing where we left off in sqliteInt.h ******************/ /* ** Include the configuration header output by 'configure' if we're using the @@ -8373,10 +8542,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 @@ -8394,22 +8563,39 @@ typedef INT16_TYPE LogEst; /* ** Macros to determine whether the machine is big or little endian, -** evaluated at runtime. +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. */ #ifdef SQLITE_AMALGAMATION SQLITE_PRIVATE const int sqlite3one = 1; #else SQLITE_PRIVATE const int sqlite3one; #endif -#if defined(i386) || defined(__i386__) || defined(_M_IX86)\ - || defined(__x86_64) || defined(__x86_64__) +#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 1234 # define SQLITE_BIGENDIAN 0 # define SQLITE_LITTLEENDIAN 1 # define SQLITE_UTF16NATIVE SQLITE_UTF16LE -#else +#endif +#if (defined(sparc) || defined(__ppc__)) \ + && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 4321 +# define SQLITE_BIGENDIAN 1 +# define SQLITE_LITTLEENDIAN 0 +# define SQLITE_UTF16NATIVE SQLITE_UTF16BE +#endif +#if !defined(SQLITE_BYTEORDER) +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) -# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) +# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) #endif /* @@ -8724,7 +8910,9 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#if SQLITE_MAX_MMAP_SIZE>0 +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); @@ -8774,6 +8962,7 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); @@ -8843,17 +9032,16 @@ SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE void sqlite3BtreeSetCachedRowid(BtCursor*, sqlite3_int64); -SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeGetCachedRowid(BtCursor*); SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *); +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); #ifndef NDEBUG SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); @@ -8984,9 +9172,12 @@ struct VdbeOp { char *zComment; /* Comment to improve readability */ #endif #ifdef VDBE_PROFILE - int cnt; /* Number of times this instruction was executed */ + u32 cnt; /* Number of times this instruction was executed */ u64 cycles; /* Total time spent executing this instruction */ #endif +#ifdef SQLITE_VDBE_COVERAGE + int iSrcLine; /* Source-code line that generated this opcode */ +#endif }; typedef struct VdbeOp VdbeOp; @@ -9090,68 +9281,68 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Checkpoint 11 #define OP_JournalMode 12 #define OP_Vacuum 13 -#define OP_VFilter 14 /* synopsis: iPlan=r[P3] zPlan='P4' */ +#define OP_VFilter 14 /* synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 15 /* synopsis: data=r[P3@P2] */ #define OP_Goto 16 #define OP_Gosub 17 #define OP_Return 18 #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Yield 20 -#define OP_HaltIfNull 21 /* synopsis: if r[P3] null then halt */ -#define OP_Halt 22 -#define OP_Integer 23 /* synopsis: r[P2]=P1 */ -#define OP_Int64 24 /* synopsis: r[P2]=P4 */ -#define OP_String 25 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 26 /* synopsis: r[P2..P3]=NULL */ -#define OP_Blob 27 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 28 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 29 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 30 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 31 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 32 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 33 -#define OP_AddImm 34 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_MustBeInt 35 -#define OP_RealAffinity 36 -#define OP_Permutation 37 -#define OP_Compare 38 -#define OP_Jump 39 -#define OP_Once 40 -#define OP_If 41 -#define OP_IfNot 42 -#define OP_Column 43 /* synopsis: r[P3]=PX */ -#define OP_Affinity 44 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 45 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 46 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 47 -#define OP_SetCookie 48 -#define OP_VerifyCookie 49 -#define OP_OpenRead 50 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 51 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenAutoindex 52 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 53 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 54 -#define OP_OpenPseudo 55 /* synopsis: content in r[P2@P3] */ -#define OP_Close 56 -#define OP_SeekLt 57 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLe 58 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGe 59 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGt 60 /* synopsis: key=r[P3@P4] */ -#define OP_Seek 61 /* synopsis: intkey=r[P2] */ -#define OP_NoConflict 62 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 63 /* synopsis: key=r[P3@P4] */ -#define OP_Found 64 /* synopsis: key=r[P3@P4] */ -#define OP_NotExists 65 /* synopsis: intkey=r[P3] */ -#define OP_Sequence 66 /* synopsis: r[P2]=rowid */ -#define OP_NewRowid 67 /* synopsis: r[P2]=rowid */ -#define OP_Insert 68 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 69 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 70 +#define OP_InitCoroutine 20 +#define OP_EndCoroutine 21 +#define OP_Yield 22 +#define OP_HaltIfNull 23 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 24 +#define OP_Integer 25 /* synopsis: r[P2]=P1 */ +#define OP_Int64 26 /* synopsis: r[P2]=P4 */ +#define OP_String 27 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 28 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 29 /* synopsis: r[P1]=NULL */ +#define OP_Blob 30 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 31 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 32 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 33 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 34 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 35 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 36 +#define OP_AddImm 37 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_MustBeInt 38 +#define OP_RealAffinity 39 +#define OP_Permutation 40 +#define OP_Compare 41 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_Jump 42 +#define OP_Once 43 +#define OP_If 44 +#define OP_IfNot 45 +#define OP_Column 46 /* synopsis: r[P3]=PX */ +#define OP_Affinity 47 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 48 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 49 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 50 +#define OP_SetCookie 51 +#define OP_ReopenIdx 52 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 53 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 54 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenAutoindex 55 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 56 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 57 +#define OP_OpenPseudo 58 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 59 +#define OP_SeekLT 60 +#define OP_SeekLE 61 +#define OP_SeekGE 62 +#define OP_SeekGT 63 +#define OP_Seek 64 /* synopsis: intkey=r[P2] */ +#define OP_NoConflict 65 /* synopsis: key=r[P3@P4] */ +#define OP_NotFound 66 /* synopsis: key=r[P3@P4] */ +#define OP_Found 67 /* synopsis: key=r[P3@P4] */ +#define OP_NotExists 68 /* synopsis: intkey=r[P3] */ +#define OP_Sequence 69 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 70 /* synopsis: r[P2]=rowid */ #define OP_Or 71 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 72 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_ResetCount 73 -#define OP_SorterCompare 74 /* synopsis: if key(P1)!=rtrim(r[P3],P4) goto P2 */ -#define OP_SorterData 75 /* synopsis: r[P2]=data */ +#define OP_Insert 73 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_InsertInt 74 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Delete 75 #define OP_IsNull 76 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 77 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 78 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ @@ -9160,7 +9351,7 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Le 81 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ #define OP_Lt 82 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ -#define OP_RowKey 84 /* synopsis: r[P2]=key */ +#define OP_ResetCount 84 #define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ #define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ #define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<0 goto P2 */ -#define OP_IfNeg 129 /* synopsis: if r[P1]<0 goto P2 */ -#define OP_IfZero 130 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ -#define OP_AggFinal 131 /* synopsis: accum=r[P1] N=P2 */ -#define OP_IncrVacuum 132 +#define OP_SorterData 98 /* synopsis: r[P2]=data */ +#define OP_RowKey 99 /* synopsis: r[P2]=key */ +#define OP_RowData 100 /* synopsis: r[P2]=data */ +#define OP_Rowid 101 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 102 +#define OP_Last 103 +#define OP_SorterSort 104 +#define OP_Sort 105 +#define OP_Rewind 106 +#define OP_SorterInsert 107 +#define OP_IdxInsert 108 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 109 /* synopsis: key=r[P2@P3] */ +#define OP_IdxRowid 110 /* synopsis: r[P2]=rowid */ +#define OP_IdxLE 111 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGT 112 /* synopsis: key=r[P3@P4] */ +#define OP_IdxLT 113 /* synopsis: key=r[P3@P4] */ +#define OP_IdxGE 114 /* synopsis: key=r[P3@P4] */ +#define OP_Destroy 115 +#define OP_Clear 116 +#define OP_ResetSorter 117 +#define OP_CreateIndex 118 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_CreateTable 119 /* synopsis: r[P2]=root iDb=P1 */ +#define OP_ParseSchema 120 +#define OP_LoadAnalysis 121 +#define OP_DropTable 122 +#define OP_DropIndex 123 +#define OP_DropTrigger 124 +#define OP_IntegrityCk 125 +#define OP_RowSetAdd 126 /* synopsis: rowset(P1)=r[P2] */ +#define OP_RowSetRead 127 /* synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 128 /* synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 129 +#define OP_Param 130 +#define OP_FkCounter 131 /* synopsis: fkctr[P1]+=P2 */ +#define OP_FkIfZero 132 /* synopsis: if fkctr[P1]==0 goto P2 */ #define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_Expire 134 -#define OP_TableLock 135 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 136 -#define OP_VCreate 137 -#define OP_VDestroy 138 -#define OP_VOpen 139 -#define OP_VColumn 140 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VNext 141 -#define OP_VRename 142 +#define OP_MemMax 134 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_IfPos 135 /* synopsis: if r[P1]>0 goto P2 */ +#define OP_IfNeg 136 /* synopsis: if r[P1]<0 goto P2 */ +#define OP_IfZero 137 /* synopsis: r[P1]+=P3, if r[P1]==0 goto P2 */ +#define OP_AggFinal 138 /* synopsis: accum=r[P1] N=P2 */ +#define OP_IncrVacuum 139 +#define OP_Expire 140 +#define OP_TableLock 141 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 142 #define OP_ToText 143 /* same as TK_TO_TEXT */ #define OP_ToBlob 144 /* same as TK_TO_BLOB */ #define OP_ToNumeric 145 /* same as TK_TO_NUMERIC */ #define OP_ToInt 146 /* same as TK_TO_INT */ #define OP_ToReal 147 /* same as TK_TO_REAL */ -#define OP_Pagecount 148 -#define OP_MaxPgcnt 149 -#define OP_Trace 150 -#define OP_Noop 151 -#define OP_Explain 152 +#define OP_VCreate 148 +#define OP_VDestroy 149 +#define OP_VOpen 150 +#define OP_VColumn 151 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VNext 152 +#define OP_VRename 153 +#define OP_Pagecount 154 +#define OP_MaxPgcnt 155 +#define OP_Init 156 /* synopsis: Start at P2 */ +#define OP_Noop 157 +#define OP_Explain 158 /* Properties such as "out2" or "jump" that are specified in @@ -9245,24 +9442,24 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01,\ /* 8 */ 0x01, 0x01, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00,\ -/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x04, 0x10, 0x00, 0x02,\ -/* 24 */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x20,\ -/* 32 */ 0x00, 0x00, 0x04, 0x05, 0x04, 0x00, 0x00, 0x01,\ -/* 40 */ 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02, 0x02,\ -/* 48 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 56 */ 0x00, 0x11, 0x11, 0x11, 0x11, 0x08, 0x11, 0x11,\ -/* 64 */ 0x11, 0x11, 0x02, 0x02, 0x00, 0x00, 0x00, 0x4c,\ +/* 16 */ 0x01, 0x01, 0x04, 0x24, 0x01, 0x04, 0x05, 0x10,\ +/* 24 */ 0x00, 0x02, 0x02, 0x02, 0x02, 0x00, 0x02, 0x02,\ +/* 32 */ 0x00, 0x00, 0x20, 0x00, 0x00, 0x04, 0x05, 0x04,\ +/* 40 */ 0x00, 0x00, 0x01, 0x01, 0x05, 0x05, 0x00, 0x00,\ +/* 48 */ 0x00, 0x02, 0x02, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11,\ +/* 64 */ 0x08, 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x4c,\ /* 72 */ 0x4c, 0x00, 0x00, 0x00, 0x05, 0x05, 0x15, 0x15,\ /* 80 */ 0x15, 0x15, 0x15, 0x15, 0x00, 0x4c, 0x4c, 0x4c,\ /* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x00,\ -/* 96 */ 0x24, 0x02, 0x02, 0x00, 0x01, 0x01, 0x01, 0x01,\ -/* 104 */ 0x08, 0x08, 0x00, 0x02, 0x01, 0x01, 0x02, 0x00,\ -/* 112 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 120 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\ -/* 128 */ 0x05, 0x05, 0x05, 0x00, 0x01, 0x02, 0x00, 0x00,\ -/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x04,\ -/* 144 */ 0x04, 0x04, 0x04, 0x04, 0x02, 0x02, 0x00, 0x00,\ -/* 152 */ 0x00,} +/* 96 */ 0x24, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01,\ +/* 104 */ 0x01, 0x01, 0x01, 0x08, 0x08, 0x00, 0x02, 0x01,\ +/* 112 */ 0x01, 0x01, 0x01, 0x02, 0x00, 0x00, 0x02, 0x02,\ +/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x45,\ +/* 128 */ 0x15, 0x01, 0x02, 0x00, 0x01, 0x02, 0x08, 0x05,\ +/* 136 */ 0x05, 0x05, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04,\ +/* 144 */ 0x04, 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x01, 0x00, 0x02, 0x02, 0x01, 0x00, 0x00,} /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -9278,7 +9475,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); @@ -9317,11 +9514,15 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); + #ifndef SQLITE_OMIT_TRIGGER SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif @@ -9349,6 +9550,43 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); # define VdbeModuleComment(X) #endif +/* +** The VdbeCoverage macros are used to set a coverage testing point +** for VDBE branch instructions. The coverage testing points are line +** numbers in the sqlite3.c source file. VDBE branch coverage testing +** only works with an amalagmation build. That's ok since a VDBE branch +** coverage build designed for testing the test suite only. No application +** should ever ship with VDBE branch coverage measuring turned on. +** +** VdbeCoverage(v) // Mark the previously coded instruction +** // as a branch +** +** VdbeCoverageIf(v, conditional) // Mark previous if conditional true +** +** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken +** +** VdbeCoverageNeverTaken(v) // Previous branch is never taken +** +** Every VDBE branch operation must be tagged with one of the macros above. +** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and +** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() +** routine in vdbe.c, alerting the developer to the missed tag. +*/ +#ifdef SQLITE_VDBE_COVERAGE +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); +# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); +# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) +#else +# define VdbeCoverage(v) +# define VdbeCoverageIf(v,x) +# define VdbeCoverageAlwaysTaken(v) +# define VdbeCoverageNeverTaken(v) +# define VDBE_OFFSET_LINENO(x) 0 +#endif + #endif /************** End of vdbe.h ************************************************/ @@ -9760,83 +9998,71 @@ SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); #define _SQLITE_OS_H_ /* -** Figure out if we are dealing with Unix, Windows, or some other -** operating system. After the following block of preprocess macros, -** all of SQLITE_OS_UNIX, SQLITE_OS_WIN, and SQLITE_OS_OTHER -** will defined to either 1 or 0. One of the four will be 1. The other -** three will be 0. +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. */ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#if SQLITE_OS_WIN -# include -#endif - +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ /* -** Determine if we are dealing with Windows NT. +** 2013 November 25 ** -** We ought to be able to determine if we are compiling for win98 or winNT -** using the _WIN32_WINNT macro as follows: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** #if defined(_WIN32_WINNT) -** # define SQLITE_OS_WINNT 1 -** #else -** # define SQLITE_OS_WINNT 0 -** #endif +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** ** -** However, vs2005 does not set _WIN32_WINNT by default, as it ought to, -** so the above test does not work. We'll just assume that everything is -** winNT unless the programmer explicitly says otherwise by setting -** SQLITE_OS_WINNT to 0. +** This file contains pre-processor directives related to operating system +** detection and/or setup. */ -#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) -# define SQLITE_OS_WINNT 1 -#endif +#ifndef _OS_SETUP_H_ +#define _OS_SETUP_H_ /* -** Determine if we are dealing with WindowsCE - which has a much -** reduced API. +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. */ -#if defined(_WIN32_WCE) -# define SQLITE_OS_WINCE 1 +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif #else -# define SQLITE_OS_WINCE 0 +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif #endif -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif +#endif /* _OS_SETUP_H_ */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op @@ -10406,8 +10632,7 @@ struct sqlite3 { ** Return true if it OK to factor constant expressions into the initialization ** code. The argument is a Parse object for the code generator. */ -#define ConstFactorOk(P) \ - ((P)->cookieGoto>0 && OptimizationEnabled((P)->db,SQLITE_FactorOutConst)) +#define ConstFactorOk(P) ((P)->okConstFactor) /* ** Possible values for the sqlite.magic field. @@ -10633,10 +10858,16 @@ struct CollSeq { /* ** Additional bit values that can be ORed with an affinity without ** changing the affinity. +** +** The SQLITE_NOTNULL flag is a combination of NULLEQ and JUMPIFNULL. +** It causes an assert() to fire if either operand to a comparison +** operator is NULL. It is added to certain comparison operators to +** prove that the operands are always NOT NULL. */ #define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ #define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ +#define SQLITE_NOTNULL 0x88 /* Assert that operands are never NULL */ /* ** An object of this type is created for each virtual table present in @@ -10730,12 +10961,15 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE @@ -10895,19 +11129,20 @@ struct KeyInfo { ** ** This structure holds a record that has already been disassembled ** into its constituent fields. +** +** The r1 and r2 member variables are only used by the optimized comparison +** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ - u8 flags; /* Boolean settings. UNPACKED_... below */ + i8 default_rc; /* Comparison result if keys are equal */ + u8 isCorrupt; /* Corruption detected by xRecordCompare() */ Mem *aMem; /* Values */ + int r1; /* Value to return if (lhs > rhs) */ + int r2; /* Value to return if (rhs < lhs) */ }; -/* -** Allowed values of UnpackedRecord.flags -*/ -#define UNPACKED_INCRKEY 0x01 /* Make this key an epsilon larger */ -#define UNPACKED_PREFIX_MATCH 0x02 /* A prefix match is considered OK */ /* ** Each SQL index is represented in memory by an @@ -10938,7 +11173,7 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ @@ -10952,7 +11187,7 @@ struct Index { u16 nKeyCol; /* Number of columns forming the key */ u16 nColumn; /* Number of columns stored in the index */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ @@ -10965,6 +11200,16 @@ struct Index { #endif }; +/* +** Allowed values for Index.idxType +*/ +#define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ +#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ +#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ + +/* Return true if index X is a PRIMARY KEY index */ +#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the @@ -11169,8 +11414,8 @@ struct Expr { #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE opeartor */ - /* unused 0x000200 */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ #define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ #define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ @@ -11234,7 +11479,6 @@ struct Expr { */ struct ExprList { int nExpr; /* Number of expressions on the list */ - int iECursor; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The list of expressions */ char *zName; /* Token associated with this expression */ @@ -11327,8 +11571,8 @@ typedef u64 Bitmask; ** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { - u8 nSrc; /* Number of tables or subqueries in the FROM clause */ - u8 nAlloc; /* Number of entries allocated in a[] below */ + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ @@ -11338,6 +11582,7 @@ struct SrcList { Select *pSelect; /* A SELECT statement used in place of a table name */ int addrFillSub; /* Address of subroutine to manifest a subquery */ int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ u8 jointype; /* Type of join between this able and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isCorrelated :1; /* True if sub-query is correlated */ @@ -11383,6 +11628,8 @@ struct SrcList { #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -11457,7 +11704,7 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ - int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -11466,7 +11713,6 @@ struct Select { ExprList *pOrderBy; /* The ORDER BY clause */ Select *pPrior; /* Prior select in a compound select statement */ Select *pNext; /* Next select to the left in a compound */ - Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ With *pWith; /* WITH clause attached to this select. Or NULL. */ @@ -11482,12 +11728,13 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x0040 /* Sort using a sorter */ + /* 0x0040 NOT USED */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ -#define SF_Materialize 0x0100 /* Force materialization of views */ + /* 0x0100 NOT USED */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ +#define SF_Compound 0x1000 /* Part of a compound query */ /* @@ -11536,13 +11783,15 @@ struct Select { ** starting with pDest->iSdst. ** ** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. +** SRT_Fifo This is like SRT_EphemTab except that the table +** is assumed to already be open. SRT_Fifo has +** the additional property of being able to ignore +** the ORDER BY clause. ** -** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** SRT_DistFifo Store results in a temporary table pDest->iSDParm. ** But also use temporary table pDest->iSDParm+1 as ** a record of all prior results and ignore any duplicate -** rows. Name means: "Distinct Table". +** rows. Name means: "Distinct Fifo". ** ** SRT_Queue Store results in priority queue pDest->iSDParm (really ** an index). Append a sequence number so that all entries @@ -11556,19 +11805,20 @@ struct Select { #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_Fifo 5 /* Store result as data with an automatic rowid */ +#define SRT_DistFifo 6 /* Like SRT_Fifo, but unique results only */ +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_DistQueue 8 /* Like SRT_Queue, but unique results only */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Output 5 /* Output each row of result */ -#define SRT_Mem 6 /* Store result in a memory cell */ -#define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 9 /* Generate a single row of result */ -#define SRT_Table 10 /* Store result as data with an automatic rowid */ -#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ -#define SRT_Queue 12 /* Store result in an queue */ -#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ /* ** An instance of this object describes where to put of the results of @@ -11636,9 +11886,19 @@ struct TriggerPrg { ** The yDbMask datatype for the bitmask of all attached databases. */ #if SQLITE_MAX_ATTACHED>30 - typedef sqlite3_uint64 yDbMask; + typedef unsigned char yDbMask[(SQLITE_MAX_ATTACHED+9)/8]; +# define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) +# define DbMaskZero(M) memset((M),0,sizeof(M)) +# define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) +# define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) #else typedef unsigned int yDbMask; +# define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) +# define DbMaskZero(M) (M)=0 +# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) +# define DbMaskAllZero(M) (M)==0 +# define DbMaskNonZero(M) (M)!=0 #endif /* @@ -11666,12 +11926,10 @@ struct Parse { u8 checkSchema; /* Causes schema cookie check after an error */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ - u8 nTempInUse; /* Number of aTempReg[] currently checked out */ - u8 nColCache; /* Number of entries in aColCache[] */ - u8 iColCache; /* Next entry in aColCache[] to replace */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ + u8 okConstFactor; /* OK to factor out constants */ int aTempReg[8]; /* Holding area for temporary registers */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -11681,30 +11939,29 @@ struct Parse { int nSet; /* Number of sets used so far */ int nOnce; /* Number of OP_Once instructions so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ - int nLabel; /* Number of labels used */ - int *aLabel; /* Space to hold the labels */ int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ int iPartIdxTab; /* Table corresponding to a partial index */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ + int nLabel; /* Number of labels used */ + int *aLabel; /* Space to hold the labels */ struct yColCache { int iTable; /* Table cursor number */ - int iColumn; /* Table column number */ + i16 iColumn; /* Table column number */ u8 tempReg; /* iReg is a temp register that needs to be freed */ int iLevel; /* Nesting level */ int iReg; /* Reg with value of this column. 0 means none. */ int lru; /* Least recently used entry has the smallest value */ } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ ExprList *pConstExpr;/* Constant expressions */ + Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ - int cookieGoto; /* Address of OP_Goto to cookie verifier subroutine */ int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ - Token constraintName;/* Name of the constraint currently being parsed */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -11723,12 +11980,17 @@ struct Parse { u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ u8 disableTriggers; /* True to disable triggers */ - /* Above is constant between recursions. Below is reset before and after - ** each recursion */ + /************************************************************************ + ** Above is constant between recursions. Below is reset before and after + ** each recursion. The boundary between these two regions is determined + ** using offsetof(Parse,nVar) so the nVar field must be the first field + ** in the recursive region. + ************************************************************************/ int nVar; /* Number of '?' variables seen in the SQL so far */ int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ + u8 bFreeWith; /* True if pWith should be freed with parser */ u8 explain; /* True if the EXPLAIN flag is found on the query */ #ifndef SQLITE_OMIT_VIRTUALTABLE u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ @@ -11755,7 +12017,6 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ - u8 bFreeWith; /* True if pWith should be freed with parser */ }; /* @@ -11962,15 +12223,25 @@ struct Sqlite3Config { int isMutexInit; /* True after mutexes are initialized */ int isMallocInit; /* True after malloc is initialized */ int isPCacheInit; /* True after malloc is initialized */ - sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ int nRefInitMutex; /* Number of users of pInitMutex */ + sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ - int bLocaltimeFault; /* True to fail localtime() calls */ #ifdef SQLITE_ENABLE_SQLLOG void(*xSqllog)(void*,sqlite3*,const char*, int); void *pSqllogArg; #endif +#ifdef SQLITE_VDBE_COVERAGE + /* The following callback (if not NULL) is invoked on every VDBE branch + ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. + */ + void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void *pVdbeBranchArg; /* 1st argument */ +#endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -12272,6 +12543,12 @@ SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); SQLITE_PRIVATE int sqlite3CodeOnce(Parse *); +#ifdef SQLITE_OMIT_BUILTIN_TEST +# define sqlite3FaultSim(X) SQLITE_OK +#else +SQLITE_PRIVATE int sqlite3FaultSim(int); +#endif + SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); @@ -12283,7 +12560,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); -SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, u8 iBatch, i64); +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); @@ -12294,6 +12571,9 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); # define sqlite3ViewGetColumnNames(A,B) 0 #endif +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); +#endif SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); @@ -12304,7 +12584,6 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); # define sqlite3AutoincrementBegin(X) # define sqlite3AutoincrementEnd(X) #endif -SQLITE_PRIVATE int sqlite3CodeCoroutine(Parse*, Select*, SelectDest*); SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int); SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); @@ -12340,6 +12619,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); SQLITE_PRIVATE u64 sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); @@ -12348,15 +12628,16 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*, int); +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); -SQLITE_PRIVATE int sqlite3ExprCode(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ @@ -12394,12 +12675,12 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*); SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); -SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(Vdbe*, const Expr*, int, int); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*); SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); @@ -12538,11 +12819,12 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); -SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *, Table *); +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE u8 sqlite3HexToInt(int h); @@ -12557,7 +12839,7 @@ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); @@ -12657,7 +12939,9 @@ SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); +SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); #endif /* @@ -13076,15 +13360,22 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* isMutexInit */ 0, /* isMallocInit */ 0, /* isPCacheInit */ - 0, /* pInitMutex */ 0, /* nRefInitMutex */ + 0, /* pInitMutex */ 0, /* xLog */ 0, /* pLogArg */ - 0, /* bLocaltimeFault */ #ifdef SQLITE_ENABLE_SQLLOG 0, /* xSqllog */ - 0 /* pSqllogArg */ + 0, /* pSqllogArg */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ #endif +#ifndef SQLITE_OMIT_BUILTIN_TEST + 0, /* xTestCallback */ +#endif + 0 /* bLocaltimeFault */ }; /* @@ -13351,6 +13642,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_COMPOUND_SELECT "OMIT_COMPOUND_SELECT", #endif +#ifdef SQLITE_OMIT_CTE + "OMIT_CTE", +#endif #ifdef SQLITE_OMIT_DATETIME_FUNCS "OMIT_DATETIME_FUNCS", #endif @@ -13639,10 +13933,11 @@ struct VdbeCursor { u8 nullRow; /* True if pointing to a row with no data */ u8 rowidIsValid; /* True if lastRowid is valid */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ Bool isTable:1; /* True if a table requiring integer keys */ Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ - Bool multiPseudo:1; /* Multi-register pseudo-cursor */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ @@ -13736,7 +14031,6 @@ struct Mem { } u; int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ - u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ @@ -13763,9 +14057,10 @@ struct Mem { #define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ +#define MEM_AffMask 0x001f /* Mask of affinity bits */ #define MEM_RowSet 0x0020 /* Value is a RowSet object */ #define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ -#define MEM_Invalid 0x0080 /* Value is undefined */ +#define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ #define MEM_TypeMask 0x01ff /* Mask of type bits */ @@ -13776,7 +14071,7 @@ struct Mem { ** string is \000 or \u0000 terminated */ #define MEM_Term 0x0200 /* String rep is nul terminated */ -#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ +#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ #define MEM_Static 0x0800 /* Mem.z points to a static string */ #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ @@ -13797,7 +14092,7 @@ struct Mem { ** is for use inside assert() statements only. */ #ifdef SQLITE_DEBUG -#define memIsValid(M) ((M)->flags & MEM_Invalid)==0 +#define memIsValid(M) ((M)->flags & MEM_Undefined)==0 #endif /* @@ -13961,7 +14256,6 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); -SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); @@ -13992,19 +14286,20 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) #define VdbeMemRelease(X) \ - if((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame)) \ - sqlite3VdbeMemReleaseExternal(X); + if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); SQLITE_PRIVATE const char *sqlite3OpcodeName(int); SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); -SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); @@ -14022,6 +14317,7 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe*,Mem*); +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY @@ -17743,7 +18039,7 @@ static void *memsys5MallocUnsafe(int nByte){ ** block. If not, then split a block of the next larger power of ** two in order to create a new free block of size iLogsize. */ - for(iBin=iLogsize; mem5.aiFreelist[iBin]<0 && iBin<=LOGMAX; iBin++){} + for(iBin=iLogsize; iBin<=LOGMAX && mem5.aiFreelist[iBin]<0; iBin++){} if( iBin>LOGMAX ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); @@ -17770,6 +18066,12 @@ static void *memsys5MallocUnsafe(int nByte){ if( mem5.maxCount +# include /* amalgamator: dontcache */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for Windows 9x or +** Windows NT using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, Visual Studio 2005 does not set _WIN32_WINNT by default, as +** it ought to, so the above test does not work. We'll just assume that +** everything is Windows NT unless the programmer explicitly says otherwise +** by setting SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with Windows CE - which has a much reduced +** API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +#endif /* _OS_WIN_H_ */ + +/************** End of os_win.h **********************************************/ +/************** Continuing where we left off in mutex_w32.c ******************/ +#endif + /* ** The code in this file is only used if we are compiling multithreaded ** on a win32 system. @@ -20040,20 +20427,6 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ -/* -** Append N space characters to the given string buffer. -*/ -SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum *pAccum, int N){ - static const char zSpaces[] = " "; - while( N>=(int)sizeof(zSpaces)-1 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1); - N -= sizeof(zSpaces)-1; - } - if( N>0 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, N); - } -} - /* ** Set the StrAccum object to an error mode. */ @@ -20143,11 +20516,9 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ - int amt; bufpt = (char *)fmt; - amt = 1; - while( (c=(*++fmt))!='%' && c!=0 ) amt++; - sqlite3StrAccumAppend(pAccum, bufpt, amt); + while( (c=(*++fmt))!='%' && c!=0 ){}; + sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); if( c==0 ) break; } if( (c=(*++fmt))==0 ){ @@ -20328,10 +20699,8 @@ SQLITE_PRIVATE void sqlite3VXPrintf( *(--bufpt) = zOrd[x*2]; } { - register const char *cset; /* Use registers for speed */ - register int base; - cset = &aDigits[infop->charset]; - base = infop->base; + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; do{ /* Convert to ascii */ *(--bufpt) = cset[longvalue%base]; longvalue = longvalue/base; @@ -20635,29 +21004,90 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** "length" characters long. The field width is "width". Do ** the output. */ - if( !flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } - if( length>0 ){ - sqlite3StrAccumAppend(pAccum, bufpt, length); - } - if( flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } + width -= length; + if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + sqlite3StrAccumAppend(pAccum, bufpt, length); + if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( zExtra ) sqlite3_free(zExtra); }/* End for loop over the format string */ } /* End of function */ /* -** Append N bytes of text from z to the StrAccum object. +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==STRACCUM_TOOBIG); + testcase(p->accError==STRACCUM_NOMEM); + return 0; + } + if( !p->useMalloc ){ + N = p->nAlloc - p->nChar - 1; + setStrAccumError(p, STRACCUM_TOOBIG); + return N; + }else{ + char *zOld = (p->zText==p->zBase ? 0 : p->zText); + i64 szNew = p->nChar; + szNew += N + 1; + if( szNew > p->mxAlloc ){ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->useMalloc==1 ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3_realloc(zOld, p->nAlloc); + } + if( zNew ){ + assert( p->zText!=0 || p->nChar==0 ); + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + }else{ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N space characters to the given string buffer. +*/ +SQLITE_PRIVATE void sqlite3AppendSpace(StrAccum *p, int N){ + if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + while( (N--)>0 ) p->zText[p->nChar++] = ' '; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3StrAccumAppend() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3StrAccumAppend() routine can use fast calling semantics. +*/ +static void enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. */ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( z!=0 ); @@ -20665,43 +21095,8 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( N>=0 ); assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ - char *zNew; - if( p->accError ){ - testcase(p->accError==STRACCUM_TOOBIG); - testcase(p->accError==STRACCUM_NOMEM); - return; - } - if( !p->useMalloc ){ - N = p->nAlloc - p->nChar - 1; - setStrAccumError(p, STRACCUM_TOOBIG); - if( N<=0 ){ - return; - } - }else{ - char *zOld = (p->zText==p->zBase ? 0 : p->zText); - i64 szNew = p->nChar; - szNew += N + 1; - if( szNew > p->mxAlloc ){ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_TOOBIG); - return; - }else{ - p->nAlloc = (int)szNew; - } - if( p->useMalloc==1 ){ - zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); - }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); - } - if( zNew ){ - if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); - p->zText = zNew; - }else{ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_NOMEM); - return; - } - } + enlargeAndAppend(p,z,N); + return; } assert( p->zText ); memcpy(&p->zText[p->nChar], z, N); @@ -21223,8 +21618,8 @@ static const unsigned char sqlite3Utf8Trans1[] = { ** and rendered as themselves even though they are technically ** invalid characters. ** -** * This routine accepts an infinite number of different UTF8 encodings -** for unicode values 0x80 and greater. It do not change over-length +** * This routine accepts over-length UTF8 encodings +** for unicode values 0x80 and greater. It does not change over-length ** encodings to 0xfffd as some systems recommend. */ #define READ_UTF8(zIn, zTerm, c) \ @@ -21392,7 +21787,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ sqlite3VdbeMemRelease(pMem); pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); pMem->enc = desiredEnc; - pMem->flags |= (MEM_Term|MEM_Dyn); + pMem->flags |= (MEM_Term); pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; @@ -21520,7 +21915,6 @@ SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 e } assert( (m.flags & MEM_Term)!=0 || db->mallocFailed ); assert( (m.flags & MEM_Str)!=0 || db->mallocFailed ); - assert( (m.flags & MEM_Dyn)!=0 || db->mallocFailed ); assert( m.z || db->mallocFailed ); return m.z; } @@ -21638,6 +22032,24 @@ SQLITE_PRIVATE void sqlite3Coverage(int x){ } #endif +/* +** Give a callback to the test harness that can be used to simulate faults +** in places where it is difficult or expensive to do so purely by means +** of inputs. +** +** The intent of the integer argument is to let the fault simulator know +** which of multiple sqlite3FaultSim() calls has been hit. +** +** Return whatever integer value the test callback returns, or return +** SQLITE_OK if no test callback is installed. +*/ +#ifndef SQLITE_OMIT_BUILTIN_TEST +SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ + int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; + return xCallback ? xCallback(iTest) : SQLITE_OK; +} +#endif + #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). @@ -22064,9 +22476,9 @@ static int compare2pow63(const char *zNum, int incr){ return c; } - /* -** Convert zNum to a 64-bit signed integer. +** Convert zNum to a 64-bit signed integer. zNum must be decimal. This +** routine does *not* accept hexadecimal notation. ** ** If the zNum value is representable as a 64-bit twos-complement ** integer, then write that value into *pNum and return 0. @@ -22154,10 +22566,44 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc } } +/* +** Transform a UTF-8 integer literal, in either decimal or hexadecimal, +** into a 64-bit signed integer. This routine accepts hexadecimal literals, +** whereas sqlite3Atoi64() does not. +** +** Returns: +** +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Integer too large for a 64-bit signed integer or is malformed +** 2 Special case of 9223372036854775808 +*/ +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) + ){ + u64 u = 0; + int i, k; + for(i=2; z[i]=='0'; i++){} + for(k=i; sqlite3Isxdigit(z[k]); k++){ + u = u*16 + sqlite3HexToInt(z[k]); + } + memcpy(pOut, &u, 8); + return (z[k]==0 && k-i<=16) ? 0 : 1; + }else +#endif /* SQLITE_OMIT_HEX_INTEGER */ + { + return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + } +} + /* ** If zNum represents an integer that will fit in 32-bits, then set ** *pValue to that integer and return true. Otherwise return false. ** +** This routine accepts both decimal and hexadecimal notation for integers. +** ** Any non-numeric characters that following zNum are ignored. ** This is different from sqlite3Atoi64() which requires the ** input number to be zero-terminated. @@ -22172,7 +22618,25 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ }else if( zNum[0]=='+' ){ zNum++; } - while( zNum[0]=='0' ) zNum++; +#ifndef SQLITE_OMIT_HEX_INTEGER + else if( zNum[0]=='0' + && (zNum[1]=='x' || zNum[1]=='X') + && sqlite3Isxdigit(zNum[2]) + ){ + u32 u = 0; + zNum += 2; + while( zNum[0]=='0' ) zNum++; + for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + u = u*16 + sqlite3HexToInt(zNum[i]); + } + if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ + memcpy(pValue, &u, 4); + return 1; + }else{ + return 0; + } + } +#endif for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ v = v*10 + c; } @@ -22730,13 +23194,12 @@ SQLITE_PRIVATE int sqlite3AddInt64(i64 *pA, i64 iB){ testcase( iA>0 && LARGEST_INT64 - iA == iB ); testcase( iA>0 && LARGEST_INT64 - iA == iB - 1 ); if( iA>0 && LARGEST_INT64 - iA < iB ) return 1; - *pA += iB; }else{ testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 1 ); testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 2 ); if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1; - *pA += iB; } + *pA += iB; return 0; } SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ @@ -22760,9 +23223,18 @@ SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ iA0 = iA % TWOPOWER32; iB1 = iB/TWOPOWER32; iB0 = iB % TWOPOWER32; - if( iA1*iB1 != 0 ) return 1; - assert( iA1*iB0==0 || iA0*iB1==0 ); - r = iA1*iB0 + iA0*iB1; + if( iA1==0 ){ + if( iB1==0 ){ + *pA *= iB; + return 0; + } + r = iA0*iB1; + }else if( iB1==0 ){ + r = iA1*iB0; + }else{ + /* If both iA1 and iB1 are non-zero, overflow will result */ + return 1; + } testcase( r==(-TWOPOWER31)-1 ); testcase( r==(-TWOPOWER31) ); testcase( r==TWOPOWER31 ); @@ -22845,8 +23317,8 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; @@ -23202,68 +23674,68 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 11 */ "Checkpoint" OpHelp(""), /* 12 */ "JournalMode" OpHelp(""), /* 13 */ "Vacuum" OpHelp(""), - /* 14 */ "VFilter" OpHelp("iPlan=r[P3] zPlan='P4'"), + /* 14 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), /* 15 */ "VUpdate" OpHelp("data=r[P3@P2]"), /* 16 */ "Goto" OpHelp(""), /* 17 */ "Gosub" OpHelp(""), /* 18 */ "Return" OpHelp(""), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Yield" OpHelp(""), - /* 21 */ "HaltIfNull" OpHelp("if r[P3] null then halt"), - /* 22 */ "Halt" OpHelp(""), - /* 23 */ "Integer" OpHelp("r[P2]=P1"), - /* 24 */ "Int64" OpHelp("r[P2]=P4"), - /* 25 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 26 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 27 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 28 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 29 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 30 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 31 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 32 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 33 */ "CollSeq" OpHelp(""), - /* 34 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 35 */ "MustBeInt" OpHelp(""), - /* 36 */ "RealAffinity" OpHelp(""), - /* 37 */ "Permutation" OpHelp(""), - /* 38 */ "Compare" OpHelp(""), - /* 39 */ "Jump" OpHelp(""), - /* 40 */ "Once" OpHelp(""), - /* 41 */ "If" OpHelp(""), - /* 42 */ "IfNot" OpHelp(""), - /* 43 */ "Column" OpHelp("r[P3]=PX"), - /* 44 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 45 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 46 */ "Count" OpHelp("r[P2]=count()"), - /* 47 */ "ReadCookie" OpHelp(""), - /* 48 */ "SetCookie" OpHelp(""), - /* 49 */ "VerifyCookie" OpHelp(""), - /* 50 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 51 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 52 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 53 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 54 */ "SorterOpen" OpHelp(""), - /* 55 */ "OpenPseudo" OpHelp("content in r[P2@P3]"), - /* 56 */ "Close" OpHelp(""), - /* 57 */ "SeekLt" OpHelp("key=r[P3@P4]"), - /* 58 */ "SeekLe" OpHelp("key=r[P3@P4]"), - /* 59 */ "SeekGe" OpHelp("key=r[P3@P4]"), - /* 60 */ "SeekGt" OpHelp("key=r[P3@P4]"), - /* 61 */ "Seek" OpHelp("intkey=r[P2]"), - /* 62 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 63 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 64 */ "Found" OpHelp("key=r[P3@P4]"), - /* 65 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 66 */ "Sequence" OpHelp("r[P2]=rowid"), - /* 67 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 68 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 69 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 70 */ "Delete" OpHelp(""), + /* 20 */ "InitCoroutine" OpHelp(""), + /* 21 */ "EndCoroutine" OpHelp(""), + /* 22 */ "Yield" OpHelp(""), + /* 23 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 24 */ "Halt" OpHelp(""), + /* 25 */ "Integer" OpHelp("r[P2]=P1"), + /* 26 */ "Int64" OpHelp("r[P2]=P4"), + /* 27 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 28 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 29 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 30 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 31 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 32 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 33 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 34 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 35 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 36 */ "CollSeq" OpHelp(""), + /* 37 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 38 */ "MustBeInt" OpHelp(""), + /* 39 */ "RealAffinity" OpHelp(""), + /* 40 */ "Permutation" OpHelp(""), + /* 41 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 42 */ "Jump" OpHelp(""), + /* 43 */ "Once" OpHelp(""), + /* 44 */ "If" OpHelp(""), + /* 45 */ "IfNot" OpHelp(""), + /* 46 */ "Column" OpHelp("r[P3]=PX"), + /* 47 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 48 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 49 */ "Count" OpHelp("r[P2]=count()"), + /* 50 */ "ReadCookie" OpHelp(""), + /* 51 */ "SetCookie" OpHelp(""), + /* 52 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 53 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 54 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 55 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 56 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 57 */ "SorterOpen" OpHelp(""), + /* 58 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 59 */ "Close" OpHelp(""), + /* 60 */ "SeekLT" OpHelp(""), + /* 61 */ "SeekLE" OpHelp(""), + /* 62 */ "SeekGE" OpHelp(""), + /* 63 */ "SeekGT" OpHelp(""), + /* 64 */ "Seek" OpHelp("intkey=r[P2]"), + /* 65 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 66 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 67 */ "Found" OpHelp("key=r[P3@P4]"), + /* 68 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 69 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 70 */ "NewRowid" OpHelp("r[P2]=rowid"), /* 71 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 72 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 73 */ "ResetCount" OpHelp(""), - /* 74 */ "SorterCompare" OpHelp("if key(P1)!=rtrim(r[P3],P4) goto P2"), - /* 75 */ "SorterData" OpHelp("r[P2]=data"), + /* 73 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 74 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 75 */ "Delete" OpHelp(""), /* 76 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 77 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 78 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), @@ -23272,7 +23744,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 81 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), /* 82 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), - /* 84 */ "RowKey" OpHelp("r[P2]=key"), + /* 84 */ "ResetCount" OpHelp(""), /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<0 goto P2"), - /* 129 */ "IfNeg" OpHelp("if r[P1]<0 goto P2"), - /* 130 */ "IfZero" OpHelp("r[P1]+=P3, if r[P1]==0 goto P2"), - /* 131 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 132 */ "IncrVacuum" OpHelp(""), + /* 98 */ "SorterData" OpHelp("r[P2]=data"), + /* 99 */ "RowKey" OpHelp("r[P2]=key"), + /* 100 */ "RowData" OpHelp("r[P2]=data"), + /* 101 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 102 */ "NullRow" OpHelp(""), + /* 103 */ "Last" OpHelp(""), + /* 104 */ "SorterSort" OpHelp(""), + /* 105 */ "Sort" OpHelp(""), + /* 106 */ "Rewind" OpHelp(""), + /* 107 */ "SorterInsert" OpHelp(""), + /* 108 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 109 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 110 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 111 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 112 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 113 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 114 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 115 */ "Destroy" OpHelp(""), + /* 116 */ "Clear" OpHelp(""), + /* 117 */ "ResetSorter" OpHelp(""), + /* 118 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), + /* 119 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), + /* 120 */ "ParseSchema" OpHelp(""), + /* 121 */ "LoadAnalysis" OpHelp(""), + /* 122 */ "DropTable" OpHelp(""), + /* 123 */ "DropIndex" OpHelp(""), + /* 124 */ "DropTrigger" OpHelp(""), + /* 125 */ "IntegrityCk" OpHelp(""), + /* 126 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 127 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 128 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 129 */ "Program" OpHelp(""), + /* 130 */ "Param" OpHelp(""), + /* 131 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 132 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), /* 133 */ "Real" OpHelp("r[P2]=P4"), - /* 134 */ "Expire" OpHelp(""), - /* 135 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 136 */ "VBegin" OpHelp(""), - /* 137 */ "VCreate" OpHelp(""), - /* 138 */ "VDestroy" OpHelp(""), - /* 139 */ "VOpen" OpHelp(""), - /* 140 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 141 */ "VNext" OpHelp(""), - /* 142 */ "VRename" OpHelp(""), + /* 134 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 135 */ "IfPos" OpHelp("if r[P1]>0 goto P2"), + /* 136 */ "IfNeg" OpHelp("if r[P1]<0 goto P2"), + /* 137 */ "IfZero" OpHelp("r[P1]+=P3, if r[P1]==0 goto P2"), + /* 138 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 139 */ "IncrVacuum" OpHelp(""), + /* 140 */ "Expire" OpHelp(""), + /* 141 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 142 */ "VBegin" OpHelp(""), /* 143 */ "ToText" OpHelp(""), /* 144 */ "ToBlob" OpHelp(""), /* 145 */ "ToNumeric" OpHelp(""), /* 146 */ "ToInt" OpHelp(""), /* 147 */ "ToReal" OpHelp(""), - /* 148 */ "Pagecount" OpHelp(""), - /* 149 */ "MaxPgcnt" OpHelp(""), - /* 150 */ "Trace" OpHelp(""), - /* 151 */ "Noop" OpHelp(""), - /* 152 */ "Explain" OpHelp(""), + /* 148 */ "VCreate" OpHelp(""), + /* 149 */ "VDestroy" OpHelp(""), + /* 150 */ "VOpen" OpHelp(""), + /* 151 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 152 */ "VNext" OpHelp(""), + /* 153 */ "VRename" OpHelp(""), + /* 154 */ "Pagecount" OpHelp(""), + /* 155 */ "MaxPgcnt" OpHelp(""), + /* 156 */ "Init" OpHelp("Start at P2"), + /* 157 */ "Noop" OpHelp(""), + /* 158 */ "Explain" OpHelp(""), }; return azName[i]; } @@ -23432,32 +23910,6 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # endif #endif -/* -** These #defines should enable >2GB file support on Posix if the -** underlying operating system supports it. If the OS lacks -** large file support, these should be no-ops. -** -** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch -** on the compiler command line. This is necessary if you are compiling -** on a recent machine (ex: RedHat 7.2) but you want your code to work -** on an older machine (ex: RedHat 6.0). If you compile on RedHat 7.2 -** without this option, LFS is enable. But LFS does not exist in the kernel -** in RedHat 6.0, so the code won't work. Hence, for maximum binary -** portability you should omit LFS. -** -** The previous paragraph was written in 2005. (This paragraph is written -** on 2008-11-28.) These days, all Linux kernels support large files, so -** you should probably leave LFS enabled. But some embedded platforms might -** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. -*/ -#ifndef SQLITE_DISABLE_LFS -# define _LARGE_FILE 1 -# ifndef _FILE_OFFSET_BITS -# define _FILE_OFFSET_BITS 64 -# endif -# define _LARGEFILE_SOURCE 1 -#endif - /* ** standard include files. */ @@ -23906,6 +24358,7 @@ static int posixFchown(int fd, uid_t uid, gid_t gid){ /* Forward reference */ static int openDirectory(const char*, int*); +static int unixGetpagesize(void); /* ** Many system calls are accessed through pointer-to-functions so that @@ -24027,6 +24480,9 @@ static struct unix_syscall { { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) + #endif }; /* End of the overrideable system calls */ @@ -25497,6 +25953,13 @@ static int closeUnixFile(sqlite3_file *id){ vxworksReleaseFileId(pFile->pId); pFile->pId = 0; } +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->zPath); + sqlite3_free(*(char**)&pFile->zPath); + pFile->zPath = 0; + } #endif OSTRACE(("CLOSE %-3d\n", pFile->h)); OpenCounter(-1); @@ -27536,8 +27999,25 @@ static int unixDeviceCharacteristics(sqlite3_file *id){ return rc; } -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + +/* +** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). +*/ +static int unixGetpagesize(void){ +#if defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +#endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ +#ifndef SQLITE_OMIT_WAL /* ** Object used to represent an shared memory buffer. @@ -27688,6 +28168,22 @@ static int unixShmSystemLock( return rc; } +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = osGetpagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ + int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->mutex); - for(i=0; inRegion; i++){ + for(i=0; inRegion; i+=nShmPerMap){ if( p->h>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ @@ -27909,6 +28406,8 @@ static int unixShmMap( unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; /* If the shared-memory file has not yet been opened, open it now. */ if( pDbFd->pShm==0 ){ @@ -27924,9 +28423,12 @@ static int unixShmMap( assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); - if( pShmNode->nRegion<=iRegion ){ + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; @@ -27975,17 +28477,19 @@ static int unixShmMap( /* Map the requested memory region into this processes address space. */ apNew = (char **)sqlite3_realloc( - pShmNode->apRegion, (iRegion+1)*sizeof(char *) + pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->apRegion = apNew; - while(pShmNode->nRegion<=iRegion){ + while( pShmNode->nRegionh>=0 ){ - pMem = osMmap(0, szRegion, + pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -28001,8 +28505,11 @@ static int unixShmMap( } memset(pMem, 0, szRegion); } - pShmNode->apRegion[pShmNode->nRegion] = pMem; - pShmNode->nRegion++; + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; } } @@ -28216,19 +28723,6 @@ static void unixUnmapfile(unixFile *pFd){ } } -/* -** Return the system page size. -*/ -static int unixGetPagesize(void){ -#if HAVE_MREMAP - return 512; -#elif defined(_BSD_SOURCE) - return getpagesize(); -#else - return (int)sysconf(_SC_PAGESIZE); -#endif -} - /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -28265,8 +28759,12 @@ static void unixRemapfile( if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; if( pOrig ){ - const int szSyspage = unixGetPagesize(); +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else + const int szSyspage = osGetpagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif u8 *pReq = &pOrig[nReuse]; /* Unmap any pages of the existing mapping that cannot be reused. */ @@ -29319,6 +29817,12 @@ static int unixOpen( if( isDelete ){ #if OS_VXWORKS zPath = zName; +#elif defined(SQLITE_UNLINK_AFTER_CLOSE) + zPath = sqlite3_mprintf("%s", zName); + if( zPath==0 ){ + robust_close(p, fd, __LINE__); + return SQLITE_NOMEM; + } #else osUnlink(zName); #endif @@ -31012,7 +31516,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==24 ); + assert( ArraySize(aSyscall)==25 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ @@ -31052,11 +31556,6 @@ SQLITE_API int sqlite3_os_end(void){ */ #if SQLITE_OS_WIN /* This file is used for Windows only */ -#ifdef __CYGWIN__ -# include -# include /* amalgamator: keep */ -#endif - /* ** Include code that is common to all os_*.c files */ @@ -31270,6 +31769,10 @@ SQLITE_API int sqlite3_open_file_count = 0; /************** End of os_common.h *******************************************/ /************** Continuing where we left off in os_win.c *********************/ +/* +** Include the header file for the Windows VFS. +*/ + /* ** Compiling and using WAL mode requires several APIs that are only ** available in Windows platforms based on the NT kernel. @@ -33081,6 +33584,32 @@ static int winLogErrorAtLine( static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + /* ** If a ReadFile() or WriteFile() error occurs, invoke this routine ** to see if it should be retried. Return TRUE to retry. Return FALSE @@ -33094,13 +33623,18 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ } return 0; } - if( e==ERROR_ACCESS_DENIED || - e==ERROR_LOCK_VIOLATION || - e==ERROR_SHARING_VIOLATION ){ + if( winIoerrCanRetry1(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } +#endif if( pError ){ *pError = e; } @@ -34039,7 +34573,7 @@ static int winGetReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } - OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); + OSTRACE(("READ-LOCK file=%p, result=%d\n", pFile->h, res)); return res; } @@ -34063,7 +34597,7 @@ static int winUnlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "winUnlockReadLock", pFile->zPath); } - OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); + OSTRACE(("READ-UNLOCK file=%p, result=%d\n", pFile->h, res)); return res; } @@ -34138,8 +34672,16 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", - pFile->h, cnt, sqlite3ErrName(res))); + lastErrno = osGetLastError(); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res)); + if( lastErrno==ERROR_INVALID_HANDLE ){ + pFile->lastErrno = lastErrno; + rc = SQLITE_IOERR_LOCK; + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc))); + return rc; + } if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -34224,7 +34766,7 @@ static int winLock(sqlite3_file *id, int locktype){ ** non-zero, otherwise zero. */ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc; + int res; winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); @@ -34232,17 +34774,17 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ - rc = 1; - OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); + res = 1; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); - if( rc ){ + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); + if( res ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } - rc = !rc; - OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); + res = !res; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (remote)\n", pFile->h, res)); } - *pResOut = rc; + *pResOut = res; OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", pFile->h, pResOut, *pResOut)); return SQLITE_OK; @@ -34383,6 +34925,17 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } +#ifdef SQLITE_TEST + case SQLITE_FCNTL_WIN32_SET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + HANDLE hOldFile = pFile->h; + pFile->h = *phFile; + *phFile = hOldFile; + OSTRACE(("FCNTL oldFile=%p, newFile=%p, rc=SQLITE_OK\n", + hOldFile, pFile->h)); + return SQLITE_OK; + } +#endif case SQLITE_FCNTL_TEMPFILENAME: { char *zTFile = 0; int rc = winGetTempname(pFile->pVfs, &zTFile); @@ -34446,7 +34999,7 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ** During sqlite3_os_init() we do a GetSystemInfo() ** to get the granularity size. */ -SYSTEM_INFO winSysInfo; +static SYSTEM_INFO winSysInfo; #ifndef SQLITE_OMIT_WAL @@ -36380,15 +36933,29 @@ static int winFullPathname( ** Interfaces for opening a shared library, finding entry points ** within the shared library, and closing the shared library. */ -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; +#if defined(__CYGWIN__) + int nFull = pVfs->mxPathname+1; + char *zFull = sqlite3MallocZero( nFull ); + void *zConverted = 0; + if( zFull==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ + sqlite3_free(zFull); + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + zConverted = winConvertFromUtf8Filename(zFull); + sqlite3_free(zFull); +#else void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); +#endif if( zConverted==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); return 0; } if( osIsNT() ){ @@ -36403,6 +36970,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ h = osLoadLibraryA((char*)zConverted); } #endif + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)h)); sqlite3_free(zConverted); return (void*)h; } @@ -36411,12 +36979,17 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); } static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ + FARPROC proc; UNUSED_PARAMETER(pVfs); - return (void(*)(void))osGetProcAddressA((HANDLE)pH, zSym); + proc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, address=%p\n", + (void*)pH, zSym, (void*)proc)); + return (void(*)(void))proc; } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); osFreeLibrary((HANDLE)pHandle); + OSTRACE(("DLCLOSE handle=%p\n", (void*)pHandle)); } #else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ #define winDlOpen 0 @@ -37112,7 +37685,8 @@ struct PCache { int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ - int bPurgeable; /* True if pages are on backing store */ + u8 bPurgeable; /* True if pages are on backing store */ + u8 eCreate; /* eCreate value for for xFetch() */ int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ void *pStress; /* Argument to xStress */ sqlite3_pcache *pCache; /* Pluggable cache module */ @@ -37179,6 +37753,10 @@ static void pcacheRemoveFromDirtyList(PgHdr *pPage){ }else{ assert( pPage==p->pDirty ); p->pDirty = pPage->pDirtyNext; + if( p->pDirty==0 && p->bPurgeable ){ + assert( p->eCreate==1 ); + p->eCreate = 2; + } } pPage->pDirtyNext = 0; pPage->pDirtyPrev = 0; @@ -37199,6 +37777,9 @@ static void pcacheAddToDirtyList(PgHdr *pPage){ if( pPage->pDirtyNext ){ assert( pPage->pDirtyNext->pDirtyPrev==0 ); pPage->pDirtyNext->pDirtyPrev = pPage; + }else if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; } p->pDirty = pPage; if( !p->pDirtyTail ){ @@ -37268,6 +37849,7 @@ SQLITE_PRIVATE void sqlite3PcacheOpen( p->szPage = szPage; p->szExtra = szExtra; p->bPurgeable = bPurgeable; + p->eCreate = 2; p->xStress = xStress; p->pStress = pStress; p->szCache = 100; @@ -37307,7 +37889,7 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( int createFlag, /* If true, create page if it does not exist already */ PgHdr **ppPage /* Write the page here */ ){ - sqlite3_pcache_page *pPage = 0; + sqlite3_pcache_page *pPage; PgHdr *pPgHdr = 0; int eCreate; @@ -37318,8 +37900,12 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( /* If the pluggable cache (sqlite3_pcache*) has not been allocated, ** allocate it now. */ - if( !pCache->pCache && createFlag ){ + if( !pCache->pCache ){ sqlite3_pcache *p; + if( !createFlag ){ + *ppPage = 0; + return SQLITE_OK; + } p = sqlite3GlobalConfig.pcache2.xCreate( pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable ); @@ -37330,11 +37916,16 @@ SQLITE_PRIVATE int sqlite3PcacheFetch( pCache->pCache = p; } - eCreate = createFlag * (1 + (!pCache->bPurgeable || !pCache->pDirty)); - if( pCache->pCache ){ - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - } - + /* eCreate defines what to do if the page does not exist. + ** 0 Do not allocate a new page. (createFlag==0) + ** 1 Allocate a new page if doing so is inexpensive. + ** (createFlag==1 AND bPurgeable AND pDirty) + ** 2 Allocate a new page even it doing so is difficult. + ** (createFlag==1 AND !(bPurgeable AND pDirty) + */ + eCreate = createFlag==0 ? 0 : pCache->eCreate; + assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate ); + pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); if( !pPage && eCreate==1 ){ PgHdr *pPg; @@ -38855,8 +39446,8 @@ struct RowSet { struct RowSetEntry *pFresh; /* Source of new entry objects */ struct RowSetEntry *pForest; /* List of binary trees of entries */ u16 nFresh; /* Number of objects on pFresh */ - u8 rsFlags; /* Various flags */ - u8 iBatch; /* Current insert batch */ + u16 rsFlags; /* Various flags */ + int iBatch; /* Current insert batch */ }; /* @@ -39190,7 +39781,7 @@ SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ ** on pRowSet->pEntry, then sort those entires into the forest at ** pRowSet->pForest so that they can be tested. */ -SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ struct RowSetEntry *p, *pTree; /* This routine is never called after sqlite3RowSetNext() */ @@ -40019,7 +40610,8 @@ struct Pager { u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ - u8 tempFile; /* zFilename is a temporary file */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ @@ -40484,7 +41076,7 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); if( isOpen(pPager->fd) ){ assert( pPager->eLock>=eLock ); - rc = sqlite3OsUnlock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); if( pPager->eLock!=UNKNOWN_LOCK ){ pPager->eLock = (u8)eLock; } @@ -40508,7 +41100,7 @@ static int pagerLockDb(Pager *pPager, int eLock){ assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); if( pPager->eLockeLock==UNKNOWN_LOCK ){ - rc = sqlite3OsLock(pPager->fd, eLock); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ pPager->eLock = (u8)eLock; IOTRACE(("LOCK %p %d\n", pPager, eLock)) @@ -41017,12 +41609,11 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_OFF + || !isOpen(pPager->jfd) ){ return SQLITE_OK; } pPager->setMaster = 1; - assert( isOpen(pPager->jfd) ); assert( pPager->journalHdr <= pPager->journalOff ); /* Calculate the length in bytes and the checksum of zMaster */ @@ -44068,30 +44659,38 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** + The value returned by sqlite3OsSectorSize() ** + The largest page size that can be written atomically. */ - if( rc==SQLITE_OK && !readOnly ){ - setSectorSize(pPager); - assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); - if( szPageDfltsectorSize ){ - if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ - szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; - }else{ - szPageDflt = (u32)pPager->sectorSize; + if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + if( !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } } - } #ifdef SQLITE_ENABLE_ATOMIC_WRITE - { - int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); - int ii; - assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); - assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); - assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); - for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ - if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ - szPageDflt = ii; + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } } } - } #endif + } + pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } } }else{ /* If a temporary file is requested, it is not opened immediately. @@ -44101,10 +44700,14 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** This branch is also run for an in-memory database. An in-memory ** database is the same as a temp-file that is never written out to ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. */ +act_like_temp_file: tempFile = 1; - pPager->eState = PAGER_READER; - pPager->eLock = EXCLUSIVE_LOCK; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE locking mode */ + pPager->noLock = 1; /* Do no locking */ readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } @@ -44145,9 +44748,6 @@ SQLITE_PRIVATE int sqlite3PagerOpen( /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ -#if 0 - assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); -#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL @@ -44282,15 +44882,17 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ - /* Check the size of the database file. If it consists of 0 pages, - ** then delete the journal file. See the header comment above for - ** the reasoning here. Delete the obsolete journal file under - ** a RESERVED lock to avoid race conditions and to avoid violating - ** [H33020]. - */ rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ - if( nPage==0 ){ + /* If the database is zero pages in size, that means that either (1) the + ** journal is a remnant from a prior database with the same name where + ** the database file but not the journal was deleted, or (2) the initial + ** transaction that populates a new database is being rolled back. + ** In either case, the journal file can be deleted. However, take care + ** not to delete the journal file if it is already open due to + ** journal_mode=PERSIST. + */ + if( nPage==0 && !jrnlOpen ){ sqlite3BeginBenignMalloc(); if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); @@ -47922,7 +48524,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( sqlite3OsClose(pRet->pWalFd); sqlite3_free(pRet); }else{ - int iDC = sqlite3OsDeviceCharacteristics(pRet->pWalFd); + int iDC = sqlite3OsDeviceCharacteristics(pDbFd); if( iDC & SQLITE_IOCAP_SEQUENTIAL ){ pRet->syncHeader = 0; } if( iDC & SQLITE_IOCAP_POWERSAFE_OVERWRITE ){ pRet->padToSectorBoundary = 0; @@ -48712,8 +49314,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** calls to sqlite3OsSleep() have a delay of 1 microsecond. Really this ** is more of a scheduler yield than an actual delay. But on the 10th ** an subsequent retries, the delays start becoming longer and longer, - ** so that on the 100th (and last) RETRY we delay for 21 milliseconds. - ** The total delay time before giving up is less than 1 second. + ** so that on the 100th (and last) RETRY we delay for 323 milliseconds. + ** The total delay time before giving up is less than 10 seconds. */ if( cnt>5 ){ int nDelay = 1; /* Pause time in microseconds */ @@ -48721,7 +49323,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ VVA_ONLY( pWal->lockError = 1; ) return SQLITE_PROTOCOL; } - if( cnt>=10 ) nDelay = (cnt-9)*238; /* Max delay 21ms. Total delay 996ms */ + if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; sqlite3OsSleep(pWal->pVfs, nDelay); } @@ -49293,7 +49895,7 @@ static int walWriteToLog( iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); - rc = sqlite3OsSync(p->pFd, p->syncFlags); + rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK); if( iAmt==0 || rc ) return rc; } rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); @@ -50227,28 +50829,30 @@ struct BtCursor { BtShared *pBt; /* The BtShared this cursor points to */ BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ -#ifndef SQLITE_OMIT_INCRBLOB Pgno *aOverflow; /* Cache of overflow page locations */ -#endif - Pgno pgnoRoot; /* The root page of this tree */ - sqlite3_int64 cachedRowid; /* Next rowid cache. 0 means not valid */ CellInfo info; /* A parse of the cell we are pointing at */ - i64 nKey; /* Size of pKey, or last integer key */ - void *pKey; /* Saved key that was cursor's last known position */ + i64 nKey; /* Size of pKey, or last integer key */ + void *pKey; /* Saved key that was cursor last known position */ + Pgno pgnoRoot; /* The root page of this tree */ + int nOvflAlloc; /* Allocated size of aOverflow[] array */ int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ - u8 wrFlag; /* True if writable */ - u8 atLast; /* Cursor pointing to the last entry */ - u8 validNKey; /* True if info.nKey is valid */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ -#ifndef SQLITE_OMIT_INCRBLOB - u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ -#endif u8 hints; /* As configured by CursorSetHints() */ i16 iPage; /* Index of current page in apPage */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; +/* +** Legal values for BtCursor.curFlags +*/ +#define BTCF_WriteFlag 0x01 /* True if a write cursor */ +#define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ + /* ** Potential values for BtCursor.eState. ** @@ -51119,16 +51723,11 @@ static int cursorHoldsMutex(BtCursor *p){ } #endif - -#ifndef SQLITE_OMIT_INCRBLOB /* -** Invalidate the overflow page-list cache for cursor pCur, if any. +** Invalidate the overflow cache of the cursor passed as the first argument. +** on the shared btree structure pBt. */ -static void invalidateOverflowCache(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - sqlite3_free(pCur->aOverflow); - pCur->aOverflow = 0; -} +#define invalidateOverflowCache(pCur) (pCur->curFlags &= ~BTCF_ValidOvfl) /* ** Invalidate the overflow page-list cache for all cursors opened @@ -51142,6 +51741,7 @@ static void invalidateAllOverflowCache(BtShared *pBt){ } } +#ifndef SQLITE_OMIT_INCRBLOB /* ** This function is called before modifying the contents of a table ** to invalidate any incrblob cursors that are open on the @@ -51164,16 +51764,14 @@ static void invalidateIncrblobCursors( BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p->isIncrblobHandle && (isClearTable || p->info.nKey==iRow) ){ + if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ p->eState = CURSOR_INVALID; } } } #else - /* Stub functions when INCRBLOB is omitted */ - #define invalidateOverflowCache(x) - #define invalidateAllOverflowCache(x) + /* Stub function when INCRBLOB is omitted */ #define invalidateIncrblobCursors(x,y,z) #endif /* SQLITE_OMIT_INCRBLOB */ @@ -51419,20 +52017,32 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ ** at is deleted out from under them. ** ** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set to one if the cursor has moved and 0 if not. +** integer *pHasMoved is set as follows: +** +** 0: The cursor is unchanged +** 1: The cursor is still pointing at the same row, but the pointers +** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() +** might now be invalid because of a balance() or other change to the +** b-tree. +** 2: The cursor is no longer pointing to the row. The row might have +** been deleted out from under the cursor. */ SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ int rc; + if( pCur->eState==CURSOR_VALID ){ + *pHasMoved = 0; + return SQLITE_OK; + } rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 1; + *pHasMoved = 2; return rc; } if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 1; + *pHasMoved = 2; }else{ - *pHasMoved = 0; + *pHasMoved = 1; } return SQLITE_OK; } @@ -52215,13 +52825,12 @@ static void zeroPage(MemPage *pPage, int flags){ memset(&data[hdr], 0, pBt->usableSize - hdr); } data[hdr] = (char)flags; - first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); + first = hdr + ((flags&PTF_LEAF)==0 ? 12 : 8); memset(&data[hdr+1], 0, 4); data[hdr+7] = 0; put2byte(&data[hdr+5], pBt->usableSize); pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); - pPage->hdrOffset = hdr; pPage->cellOffset = first; pPage->aDataEnd = &data[pBt->usableSize]; pPage->aCellIdx = &data[first]; @@ -52835,6 +53444,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be ** memory mapped. @@ -52847,6 +53457,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ sqlite3BtreeLeave(p); return SQLITE_OK; } +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** Change the way data is synced to disk in order to increase or decrease @@ -53223,7 +53834,8 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ BtCursor *pCur; int r = 0; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) + && pCur->eState!=CURSOR_FAULT ) r++; } return r; } @@ -54298,14 +54910,14 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->wrFlag = (u8)wrFlag; + assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); + pCur->curFlags = wrFlag; pCur->pNext = pBt->pCursor; if( pCur->pNext ){ pCur->pNext->pPrev = pCur; } pBt->pCursor = pCur; pCur->eState = CURSOR_INVALID; - pCur->cachedRowid = 0; return SQLITE_OK; } SQLITE_PRIVATE int sqlite3BtreeCursor( @@ -54346,36 +54958,6 @@ SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ memset(p, 0, offsetof(BtCursor, iPage)); } -/* -** Set the cached rowid value of every cursor in the same database file -** as pCur and having the same root page number as pCur. The value is -** set to iRowid. -** -** Only positive rowid values are considered valid for this cache. -** The cache is initialized to zero, indicating an invalid cache. -** A btree will work fine with zero or negative rowids. We just cannot -** cache zero or negative rowids, which means tables that use zero or -** negative rowids might run a little slower. But in practice, zero -** or negative rowids are very uncommon so this should not be a problem. -*/ -SQLITE_PRIVATE void sqlite3BtreeSetCachedRowid(BtCursor *pCur, sqlite3_int64 iRowid){ - BtCursor *p; - for(p=pCur->pBt->pCursor; p; p=p->pNext){ - if( p->pgnoRoot==pCur->pgnoRoot ) p->cachedRowid = iRowid; - } - assert( pCur->cachedRowid==iRowid ); -} - -/* -** Return the cached rowid for the given cursor. A negative or zero -** return value indicates that the rowid cache is invalid and should be -** ignored. If the rowid cache has never before been set, then a -** zero is returned. -*/ -SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeGetCachedRowid(BtCursor *pCur){ - return pCur->cachedRowid; -} - /* ** Close a cursor. The read lock on the database file is released ** when the last cursor is closed. @@ -54399,7 +54981,7 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - invalidateOverflowCache(pCur); + sqlite3DbFree(pBtree->db, pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -54438,7 +55020,7 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ if( pCur->info.nSize==0 ){ int iPage = pCur->iPage; btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->validNKey = 1; + pCur->curFlags |= BTCF_ValidNKey; }else{ assertCellInfo(pCur); } @@ -54448,8 +55030,8 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ #define getCellInfo(pCur) \ if( pCur->info.nSize==0 ){ \ int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->validNKey = 1; \ + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->curFlags |= BTCF_ValidNKey; \ }else{ \ assertCellInfo(pCur); \ } @@ -54620,10 +55202,12 @@ static int copyPayload( /* ** This function is used to read or overwrite payload information -** for the entry that the pCur cursor is pointing to. If the eOp -** parameter is 0, this is a read operation (data copied into -** buffer pBuf). If it is non-zero, a write (data copied from -** buffer pBuf). +** for the entry that the pCur cursor is pointing to. The eOp +** argument is interpreted as follows: +** +** 0: The operation is a read. Populate the overflow cache. +** 1: The operation is a write. Populate the overflow cache. +** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -54631,11 +55215,11 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the BtCursor.isIncrblobHandle flag is set, and the current -** cursor entry uses one or more overflow pages, this function -** allocates space for and lazily popluates the overflow page-list -** cache array (BtCursor.aOverflow). Subsequent calls use this -** cache to make seeking to the supplied offset more efficient. +** If the current cursor entry uses one or more overflow pages and the +** eOp argument is not 2, this function may allocate space for and lazily +** popluates the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset +** more efficient. ** ** Once an overflow page-list cache has been allocated, it may be ** invalidated if some other cursor writes to the same table, or if @@ -54659,15 +55243,22 @@ static int accessPayload( int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + int bEnd; /* True if reading to end of data */ +#endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); +#ifdef SQLITE_DIRECT_OVERFLOW_READ + bEnd = (offset+amt==nKey+pCur->info.nData); +#endif if( NEVER(offset+amt > nKey+pCur->info.nData) || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] @@ -54682,7 +55273,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -54696,21 +55287,30 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); -#ifndef SQLITE_OMIT_INCRBLOB - /* If the isIncrblobHandle flag is set and the BtCursor.aOverflow[] - ** has not been allocated, allocate it now. The array is sized at - ** one entry for each overflow page in the overflow chain. The - ** page number of the first overflow page is stored in aOverflow[0], - ** etc. A value of 0 in the aOverflow[] array means "not yet known" - ** (the cache is lazily populated). + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. + ** Except, do not allocate aOverflow[] for eOp==2. + ** + ** The aOverflow[] array is sized at one entry for each overflow page + ** in the overflow chain. The page number of the first overflow page is + ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array + ** means "not yet known" (the cache is lazily populated). */ - if( pCur->isIncrblobHandle && !pCur->aOverflow ){ + if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - pCur->aOverflow = (Pgno *)sqlite3MallocZero(sizeof(Pgno)*nOvfl); - /* nOvfl is always positive. If it were zero, fetchPayload would have - ** been used instead of this routine. */ - if( ALWAYS(nOvfl) && !pCur->aOverflow ){ - rc = SQLITE_NOMEM; + if( nOvfl>pCur->nOvflAlloc ){ + Pgno *aNew = (Pgno*)sqlite3DbRealloc( + pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pCur->nOvflAlloc = nOvfl*2; + pCur->aOverflow = aNew; + } + } + if( rc==SQLITE_OK ){ + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; } } @@ -54718,22 +55318,19 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( pCur->aOverflow && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); } -#endif for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ -#ifndef SQLITE_OMIT_INCRBLOB /* If required, populate the overflow page-list cache. */ - if( pCur->aOverflow ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); pCur->aOverflow[iIdx] = nextPage; } -#endif if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -54741,13 +55338,17 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. + ** + ** Note that the aOverflow[] array must be allocated because eOp!=2 + ** here. If eOp==2, then offset==0 and this branch is never taken. */ -#ifndef SQLITE_OMIT_INCRBLOB - if( pCur->aOverflow && pCur->aOverflow[iIdx+1] ){ + assert( eOp!=2 ); + assert( pCur->curFlags & BTCF_ValidOvfl ); + if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; - } else -#endif + }else{ rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + } offset -= ovflSize; }else{ /* Need to read this page properly. It contains some of the @@ -54769,13 +55370,15 @@ static int accessPayload( ** 3) the database is file-backed, and ** 4) there is no open write-transaction, and ** 5) the database is not a WAL database, + ** 6) all data from the page is being read. ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( eOp==0 /* (1) */ + if( (eOp&0x01)==0 /* (1) */ && offset==0 /* (2) */ + && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ && pBt->pPage1->aData[19]==0x01 /* (5) */ @@ -54792,12 +55395,12 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, - (eOp==0 ? PAGER_GET_READONLY : 0) + ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } @@ -54891,10 +55494,7 @@ static const void *fetchPayload( assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorHoldsMutex(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - if( pCur->info.nSize==0 ){ - btreeParseCell(pCur->apPage[pCur->iPage], pCur->aiIdx[pCur->iPage], - &pCur->info); - } + assert( pCur->info.nSize>0 ); *pAmt = pCur->info.nLocal; return (void*)(pCur->info.pCell + pCur->info.nHeader); } @@ -54945,14 +55545,14 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ return SQLITE_CORRUPT_BKPT; } rc = getAndInitPage(pBt, newPgno, &pNewPage, - pCur->wrFlag==0 ? PAGER_GET_READONLY : 0); + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; pCur->iPage++; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ return SQLITE_CORRUPT_BKPT; } @@ -55010,7 +55610,7 @@ static void moveToParent(BtCursor *pCur){ releasePage(pCur->apPage[pCur->iPage]); pCur->iPage--; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); } /* @@ -55057,7 +55657,7 @@ static int moveToRoot(BtCursor *pCur){ return SQLITE_OK; }else{ rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - pCur->wrFlag==0 ? PAGER_GET_READONLY : 0); + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -55084,8 +55684,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->aiIdx[0] = 0; pCur->info.nSize = 0; - pCur->atLast = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); if( pRoot->nCell>0 ){ pCur->eState = CURSOR_VALID; @@ -55148,7 +55747,7 @@ static int moveToRightmost(BtCursor *pCur){ if( rc==SQLITE_OK ){ pCur->aiIdx[pCur->iPage] = pPage->nCell-1; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~BTCF_ValidNKey; } return rc; } @@ -55187,7 +55786,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ - if( CURSOR_VALID==pCur->eState && pCur->atLast ){ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ #ifdef SQLITE_DEBUG /* This block serves to assert() that the cursor really does point ** to the last entry in the b-tree. */ @@ -55210,7 +55809,12 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( pCur->eState==CURSOR_VALID ); *pRes = 0; rc = moveToRightmost(pCur); - pCur->atLast = rc==SQLITE_OK ?1:0; + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + } } return rc; @@ -55252,6 +55856,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( int *pRes /* Write search results here */ ){ int rc; + RecordCompare xRecordCompare; assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -55260,19 +55865,30 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && pCur->validNKey + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 && pCur->apPage[0]->intKey ){ if( pCur->info.nKey==intKey ){ *pRes = 0; return SQLITE_OK; } - if( pCur->atLast && pCur->info.nKeycurFlags & BTCF_AtLast)!=0 && pCur->info.nKeyisCorrupt = 0; + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); + }else{ + xRecordCompare = 0; /* All keys are integers */ + } + rc = moveToRoot(pCur); if( rc ){ return rc; @@ -55305,7 +55921,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( assert( biasRight==0 || biasRight==1 ); idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pPage->intKey ){ + if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; pCell = findCell(pPage, idx) + pPage->childPtrSize; @@ -55323,7 +55939,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ c = +1; break; } }else{ assert( nCellKey==intKey ); - pCur->validNKey = 1; + pCur->curFlags |= BTCF_ValidNKey; pCur->info.nKey = nCellKey; pCur->aiIdx[pCur->iPage] = (u16)idx; if( !pPage->leaf ){ @@ -55357,14 +55973,14 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -55380,14 +55996,15 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } - c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } + assert( pIdxKey->isCorrupt==0 || c==0 ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -55397,6 +56014,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; + if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; @@ -55425,7 +56043,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( } moveto_finish: pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); return rc; } @@ -55450,6 +56068,15 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ ** successful then set *pRes=0. If the cursor ** was already pointing to the last entry in the database before ** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) */ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ int rc; @@ -55458,8 +56085,10 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); if( pCur->eState!=CURSOR_VALID ){ + invalidateOverflowCache(pCur); rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ *pRes = 0; @@ -55493,7 +56122,7 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ testcase( idx>pPage->nCell ); pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); @@ -55536,6 +56165,15 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** successful then set *pRes=0. If the cursor ** was already pointing to the first entry in the database before ** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) */ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ int rc; @@ -55543,8 +56181,9 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - pCur->atLast = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); if( pCur->eState!=CURSOR_VALID ){ if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ rc = btreeRestoreCursorPosition(pCur); @@ -55589,7 +56228,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ moveToParent(pCur); } pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -57614,7 +58253,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -57639,11 +58278,17 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; - /* If this is an insert into a table b-tree, invalidate any incrblob - ** cursors open on the row being replaced (assuming this is a replace - ** operation - if it is not, the following is a no-op). */ if( pCur->pKeyInfo==0 ){ + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced */ invalidateIncrblobCursors(p, nKey, 0); + + /* If the cursor is currently on the last row and we are appending a + ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() + ** call */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ + loc = -1; + } } if( !loc ){ @@ -57694,7 +58339,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move - ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey + ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey ** variables. ** ** Previous versions of SQLite called moveToRoot() to move the cursor @@ -57713,8 +58358,8 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** row without seeking the cursor. This can be a big performance boost. */ pCur->info.nSize = 0; - pCur->validNKey = 0; if( rc==SQLITE_OK && pPage->nOverflow ){ + pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -57746,7 +58391,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); - assert( pCur->wrFlag ); + assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); @@ -57769,7 +58414,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur){ ** sub-tree headed by the child page of the cell being deleted. This makes ** balancing the tree following the delete operation easier. */ if( !pPage->leaf ){ - int notUsed; + int notUsed = 0; rc = sqlite3BtreePrevious(pCur, ¬Used); if( rc ) return rc; } @@ -58091,6 +58736,15 @@ SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ return rc; } +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + /* ** Erase all information in a table and add the root of the table to ** the freelist. Except, the root of the principle table (the one on @@ -59050,7 +59704,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void int rc; assert( cursorHoldsMutex(pCsr) ); assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); - assert( pCsr->isIncrblobHandle ); + assert( pCsr->curFlags & BTCF_Incrblob ); rc = restoreCursorPosition(pCsr); if( rc!=SQLITE_OK ){ @@ -59079,7 +59733,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void ** (d) there are no conflicting read-locks, and ** (e) the cursor points at a valid row of an intKey table. */ - if( !pCsr->wrFlag ){ + if( (pCsr->curFlags & BTCF_WriteFlag)==0 ){ return SQLITE_READONLY; } assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 @@ -59092,20 +59746,10 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void } /* -** Set a flag on this cursor to cache the locations of pages from the -** overflow list for the current row. This is used by cursors opened -** for incremental blob IO only. -** -** This function sets a flag only. The actual page location cache -** (stored in BtCursor.aOverflow[]) is allocated and used by function -** accessPayload() (the worker function for sqlite3BtreeData() and -** sqlite3BtreePutData()). +** Mark this cursor as an incremental blob cursor. */ -SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - invalidateOverflowCache(pCur); - pCur->isIncrblobHandle = 1; +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ + pCur->curFlags |= BTCF_Incrblob; } #endif @@ -59154,6 +59798,13 @@ SQLITE_PRIVATE void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ pCsr->hints = mask; } +/* +** Return true if the given Btree is read-only. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ + return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; +} + /************** End of btree.c ***********************************************/ /************** Begin file backup.c ******************************************/ /* @@ -59923,6 +60574,42 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ ** name sqlite_value */ +#ifdef SQLITE_DEBUG +/* +** Check invariants on a Mem object. +** +** This routine is intended for use inside of assert() statements, like +** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ + /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor + ** function for Mem.z + */ + assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); + + /* If p holds a string or blob, the Mem.z must point to exactly + ** one of the following: + ** + ** (1) Memory in Mem.zMalloc and managed by the Mem object + ** (2) Memory to be freed using Mem.xDel + ** (3) An ephermal string or blob + ** (4) A static string or blob + */ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + assert( + ((p->z==p->zMalloc)? 1 : 0) + + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 + ); + } + + return 1; +} +#endif + + /* ** If pMem is an object with a valid string representation, this routine ** ensures the internal encoding for the string representation is @@ -59972,12 +60659,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** in pMem->z is discarded. */ SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ - assert( 1 >= - ((pMem->zMalloc && pMem->zMalloc==pMem->z) ? 1 : 0) + - (((pMem->flags&MEM_Dyn)&&pMem->xDel) ? 1 : 0) + - ((pMem->flags&MEM_Ephem) ? 1 : 0) + - ((pMem->flags&MEM_Static) ? 1 : 0) - ); + assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -59995,7 +60677,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - sqlite3VdbeMemRelease(pMem); + VdbeMemRelease(pMem); + pMem->z = 0; pMem->flags = MEM_Null; return SQLITE_NOMEM; } @@ -60004,13 +60687,13 @@ SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } - if( (pMem->flags&MEM_Dyn)!=0 && pMem->xDel ){ - assert( pMem->xDel!=SQLITE_DYNAMIC ); + if( (pMem->flags&MEM_Dyn)!=0 ){ + assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC ); pMem->xDel((void *)(pMem->z)); } pMem->z = pMem->zMalloc; - pMem->flags &= ~(MEM_Ephem|MEM_Static); + pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); pMem->xDel = 0; return SQLITE_OK; } @@ -60179,9 +60862,9 @@ SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ sqlite3VdbeMemFinalize(p, p->u.pDef); assert( (p->flags & MEM_Agg)==0 ); sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn && p->xDel ){ + }else if( p->flags&MEM_Dyn ){ assert( (p->flags&MEM_RowSet)==0 ); - assert( p->xDel!=SQLITE_DYNAMIC ); + assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ @@ -60194,9 +60877,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p){ /* ** Release any memory held by the Mem. This may leave the Mem in an ** inconsistent state, for example with (Mem.z==0) and -** (Mem.type==SQLITE_TEXT). +** (Mem.flags==MEM_Str). */ SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ + assert( sqlite3VdbeCheckMemInvariants(p) ); VdbeMemRelease(p); if( p->zMalloc ){ sqlite3DbFree(p->db, p->zMalloc); @@ -60385,7 +61069,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ sqlite3RowSetClear(pMem->u.pRowSet); } MemSetTypeFlag(pMem, MEM_Null); - pMem->type = SQLITE_NULL; } SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ sqlite3VdbeMemSetNull((Mem*)p); @@ -60398,7 +61081,6 @@ SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ sqlite3VdbeMemRelease(pMem); pMem->flags = MEM_Blob|MEM_Zero; - pMem->type = SQLITE_BLOB; pMem->n = 0; if( n<0 ) n = 0; pMem->u.nZero = n; @@ -60421,7 +61103,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ sqlite3VdbeMemRelease(pMem); pMem->u.i = val; pMem->flags = MEM_Int; - pMem->type = SQLITE_INTEGER; } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -60436,7 +61117,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ sqlite3VdbeMemRelease(pMem); pMem->r = val; pMem->flags = MEM_Real; - pMem->type = SQLITE_FLOAT; } } #endif @@ -60492,7 +61172,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ Mem *pX; for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ - pX->flags |= MEM_Invalid; + pX->flags |= MEM_Undefined; pX->pScopyFrom = 0; } } @@ -60534,6 +61214,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ VdbeMemRelease(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; + pTo->xDel = 0; if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ @@ -60644,7 +61325,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( pMem->n = nByte; pMem->flags = flags; pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); - pMem->type = (enc==0 ? SQLITE_BLOB : SQLITE_TEXT); #ifndef SQLITE_OMIT_UTF16 if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ @@ -60659,119 +61339,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( return SQLITE_OK; } -/* -** Compare the values contained by the two memory cells, returning -** negative, zero or positive if pMem1 is less than, equal to, or greater -** than pMem2. Sorting order is NULL's first, followed by numbers (integers -** and reals) sorted numerically, followed by text ordered by the collating -** sequence pColl and finally blob's ordered by memcmp(). -** -** Two NULL values are considered equal by this function. -*/ -SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; - int f1, f2; - int combined_flags; - - f1 = pMem1->flags; - f2 = pMem2->flags; - combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); - - /* If one value is NULL, it is less than the other. If both values - ** are NULL, return 0. - */ - if( combined_flags&MEM_Null ){ - return (f2&MEM_Null) - (f1&MEM_Null); - } - - /* If one value is a number and the other is not, the number is less. - ** If both are numbers, compare as reals if one is a real, or as integers - ** if both values are integers. - */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - double r1, r2; - if( (f1 & f2 & MEM_Int)!=0 ){ - if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return 1; - return 0; - } - if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->r; - }else if( (f1&MEM_Int)!=0 ){ - r1 = (double)pMem1->u.i; - }else{ - return 1; - } - if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->r; - }else if( (f2&MEM_Int)!=0 ){ - r2 = (double)pMem2->u.i; - }else{ - return -1; - } - if( r1r2 ) return 1; - return 0; - } - - /* If one value is a string and the other is a blob, the string is less. - ** If both are strings, compare using the collating functions. - */ - if( combined_flags&MEM_Str ){ - if( (f1 & MEM_Str)==0 ){ - return 1; - } - if( (f2 & MEM_Str)==0 ){ - return -1; - } - - assert( pMem1->enc==pMem2->enc ); - assert( pMem1->enc==SQLITE_UTF8 || - pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); - - /* The collation sequence must be defined at this point, even if - ** the user deletes the collation sequence after the vdbe program is - ** compiled (this was not always the case). - */ - assert( !pColl || pColl->xCmp ); - - if( pColl ){ - if( pMem1->enc==pColl->enc ){ - /* The strings are already in the correct encoding. Call the - ** comparison function directly */ - return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); - }else{ - const void *v1, *v2; - int n1, n2; - Mem c1; - Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - n1 = v1==0 ? 0 : c1.n; - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - n2 = v2==0 ? 0 : c2.n; - rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); - sqlite3VdbeMemRelease(&c1); - sqlite3VdbeMemRelease(&c2); - return rc; - } - } - /* If a NULL pointer was passed as the collate function, fall through - ** to the blob case and use memcmp(). */ - } - - /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; - } - return rc; -} - /* ** Move data out of a btree key or data field and into a Mem structure. ** The data or key is taken from the entry that pCur is currently pointing @@ -60812,22 +61379,22 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( sqlite3VdbeMemRelease(pMem); pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - pMem->flags = MEM_Blob|MEM_Dyn|MEM_Term; - pMem->enc = 0; - pMem->type = SQLITE_BLOB; if( key ){ rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); }else{ rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); } - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ sqlite3VdbeMemRelease(pMem); } } - pMem->n = (int)amt; return rc; } @@ -60885,7 +61452,6 @@ SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); if( p ){ p->flags = MEM_Null; - p->type = SQLITE_NULL; p->db = db; } return p; @@ -60931,11 +61497,9 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ if( pRec->pKeyInfo ){ assert( pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField==nCol ); assert( pRec->pKeyInfo->enc==ENC(db) ); - pRec->flags = UNPACKED_PREFIX_MATCH; pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); for(i=0; iaMem[i].flags = MEM_Null; - pRec->aMem[i].type = SQLITE_NULL; pRec->aMem[i].db = db; } }else{ @@ -61008,7 +61572,6 @@ static int valueFromExpr( zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); - if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); @@ -61026,9 +61589,9 @@ static int valueFromExpr( ){ sqlite3VdbeMemNumerify(pVal); if( pVal->u.i==SMALLEST_INT64 ){ - pVal->flags &= MEM_Int; + pVal->flags &= ~MEM_Int; pVal->flags |= MEM_Real; - pVal->r = (double)LARGEST_INT64; + pVal->r = (double)SMALLEST_INT64; }else{ pVal->u.i = -pVal->u.i; } @@ -61054,9 +61617,6 @@ static int valueFromExpr( } #endif - if( pVal ){ - sqlite3VdbeMemStoreType(pVal); - } *ppVal = pVal; return rc; @@ -61148,6 +61708,68 @@ SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ } } +/* +** Attempt to extract a value from pExpr and use it to construct *ppVal. +** +** If pAlloc is not NULL, then an UnpackedRecord object is created for +** pAlloc if one does not exist and the new value is added to the +** UnpackedRecord object. +** +** A value is extracted in the following cases: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The expression is a literal value. +** +** On success, *ppVal is made to point to the extracted value. The caller +** is responsible for ensuring that the value is eventually freed. +*/ +static int stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + struct ValueNewStat4Ctx *pAlloc,/* How to allocate space. Or NULL */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + int rc = SQLITE_OK; + sqlite3_value *pVal = 0; + sqlite3 *db = pParse->db; + + /* Skip over any TK_COLLATE nodes */ + pExpr = sqlite3ExprSkipCollate(pExpr); + + if( !pExpr ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + } + }else if( pExpr->op==TK_VARIABLE + || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + Vdbe *v; + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); + } + pVal->db = pParse->db; + } + } + }else{ + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, pAlloc); + } + + assert( pVal==0 || pVal->db==db ); + *ppVal = pVal; + return rc; +} + /* ** This function is used to allocate and populate UnpackedRecord ** structures intended to be compared against sample index keys stored @@ -61187,51 +61809,88 @@ SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( int iVal, /* Array element to populate */ int *pbOk /* OUT: True if value was extracted */ ){ - int rc = SQLITE_OK; + int rc; sqlite3_value *pVal = 0; - sqlite3 *db = pParse->db; - - struct ValueNewStat4Ctx alloc; + alloc.pParse = pParse; alloc.pIdx = pIdx; alloc.ppRec = ppRec; alloc.iVal = iVal; - /* Skip over any TK_COLLATE nodes */ - pExpr = sqlite3ExprSkipCollate(pExpr); - - if( !pExpr ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - sqlite3VdbeMemSetNull((Mem*)pVal); - } - }else if( pExpr->op==TK_VARIABLE - || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ - Vdbe *v; - int iBindVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); - if( (v = pParse->pReprepare)!=0 ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); - } - pVal->db = pParse->db; - sqlite3VdbeMemStoreType((Mem*)pVal); - } - } - }else{ - rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, &alloc); - } + rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); + assert( pVal==0 || pVal->db==pParse->db ); *pbOk = (pVal!=0); - - assert( pVal==0 || pVal->db==db ); return rc; } +/* +** Attempt to extract a value from expression pExpr using the methods +** as described for sqlite3Stat4ProbeSetValue() above. +** +** If successful, set *ppVal to point to a new value object and return +** SQLITE_OK. If no value can be extracted, but no other error occurs +** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error +** does occur, return an SQLite error code. The final value of *ppVal +** is undefined in this case. +*/ +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); +} + +/* +** Extract the iCol-th column from the nRec-byte record in pRec. Write +** the column value into *ppVal. If *ppVal is initially NULL then a new +** sqlite3_value object is allocated. +** +** If *ppVal is initially NULL then the caller is responsible for +** ensuring that the value written into *ppVal is eventually freed. +*/ +SQLITE_PRIVATE int sqlite3Stat4Column( + sqlite3 *db, /* Database handle */ + const void *pRec, /* Pointer to buffer containing record */ + int nRec, /* Size of buffer pRec in bytes */ + int iCol, /* Column to extract */ + sqlite3_value **ppVal /* OUT: Extracted value */ +){ + u32 t; /* a column type code */ + int nHdr; /* Size of the header in the record */ + int iHdr; /* Next unread header byte */ + int iField; /* Next unread data byte */ + int szField; /* Size of the current data field */ + int i; /* Column index */ + u8 *a = (u8*)pRec; /* Typecast byte array */ + Mem *pMem = *ppVal; /* Write result into this Mem object */ + + assert( iCol>0 ); + iHdr = getVarint32(a, nHdr); + if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + iField = nHdr; + for(i=0; i<=iCol; i++){ + iHdr += getVarint32(&a[iHdr], t); + testcase( iHdr==nHdr ); + testcase( iHdr==nHdr+1 ); + if( iHdr>nHdr ) return SQLITE_CORRUPT_BKPT; + szField = sqlite3VdbeSerialTypeLen(t); + iField += szField; + } + testcase( iField==nRec ); + testcase( iField==nRec+1 ); + if( iField>nRec ) return SQLITE_CORRUPT_BKPT; + if( pMem==0 ){ + pMem = *ppVal = sqlite3ValueNew(db); + if( pMem==0 ) return SQLITE_NOMEM; + } + sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); + pMem->enc = ENC(db); + return SQLITE_OK; +} + /* ** Unless it is NULL, the argument must be an UnpackedRecord object returned ** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes @@ -61465,6 +62124,9 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #ifdef VDBE_PROFILE pOp->cycles = 0; pOp->cnt = 0; +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; #endif return i; } @@ -61565,7 +62227,7 @@ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ int j = -1-x; assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - if( j>=0 && p->aLabel ){ + if( ALWAYS(j>=0) && p->aLabel ){ p->aLabel[j] = v->nOp; } p->iFixedOp = v->nOp - 1; @@ -61788,7 +62450,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pParse->aLabel = 0; pParse->nLabel = 0; *pMaxFuncArgs = nMaxArgs; - assert( p->bIsReader!=0 || p->btreeMask==0 ); + assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } /* @@ -61815,7 +62477,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) assert( aOp && !p->db->mallocFailed ); /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ - assert( p->btreeMask==0 ); + assert( DbMaskAllZero(p->btreeMask) ); resolveP2Values(p, pnMaxArg); *pnOp = p->nOp; @@ -61827,7 +62489,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg) ** Add a whole list of operations to the operation stack. Return the ** address of the first operation added. */ -SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){ +SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ int addr; assert( p->magic==VDBE_MAGIC_INIT ); if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p) ){ @@ -61855,6 +62517,11 @@ SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp) #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS pOut->zComment = 0; #endif +#ifdef SQLITE_VDBE_COVERAGE + pOut->iSrcLine = iLineno+i; +#else + (void)iLineno; +#endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ sqlite3VdbePrintOp(0, i+addr, &p->aOp[i+addr]); @@ -62067,7 +62734,9 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED || pOp->p4type==P4_INT32 ); + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_INT32 + || pOp->p4type==P4_KEYINFO ); freeP4(db, pOp->p4type, pOp->p4.p); pOp->p4.p = 0; if( n==P4_INT32 ){ @@ -62143,6 +62812,15 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ } #endif /* NDEBUG */ +#ifdef SQLITE_VDBE_COVERAGE +/* +** Set the value if the iSrcLine field for the previously coded instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ + sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; +} +#endif /* SQLITE_VDBE_COVERAGE */ + /* ** Return the opcode for a given address. If the address is -1, then ** return the most recently inserted opcode. @@ -62155,14 +62833,6 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** this routine is a valid pointer. But because the dummy.opcode is 0, ** dummy will never be written to. This is verified by code inspection and ** by running with Valgrind. -** -** About the #ifdef SQLITE_OMIT_TRACE: Normally, this routine is never called -** unless p->nOp>0. This is because in the absense of SQLITE_OMIT_TRACE, -** an OP_Trace instruction is always inserted by sqlite3VdbeGet() as soon as -** a new VDBE is created. So we are free to set addr to p->nOp-1 without -** having to double-check to make sure that the result is non-negative. But -** if SQLITE_OMIT_TRACE is defined, the OP_Trace is omitted and we do need to -** check the value of p->nOp-1 before continuing. */ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* C89 specifies that the constant "dummy" will be initialized to all @@ -62170,9 +62840,6 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ assert( p->magic==VDBE_MAGIC_INIT ); if( addr<0 ){ -#ifdef SQLITE_OMIT_TRACE - if( p->nOp==0 ) return (VdbeOp*)&dummy; -#endif addr = p->nOp - 1; } assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); @@ -62395,9 +63062,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); assert( i<(int)sizeof(p->btreeMask)*8 ); - p->btreeMask |= ((yDbMask)1)<btreeMask, i); if( i!=1 && sqlite3BtreeSharable(p->db->aDb[i].pBt) ){ - p->lockMask |= ((yDbMask)1)<lockMask, i); } } @@ -62425,16 +63092,15 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ */ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeEnter(aDb[i].pBt); } } @@ -62447,16 +63113,15 @@ SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ */ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeLeave(aDb[i].pBt); } } @@ -62477,7 +63142,7 @@ SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS displayComment(pOp, zP4, zCom, sizeof(zCom)); #else - zCom[0] = 0 + zCom[0] = 0; #endif /* NB: The sqlite3OpcodeName() function is implemented by code created ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the @@ -62506,6 +63171,7 @@ static void releaseMemArray(Mem *p, int N){ } for(pEnd=&p[N]; pflags & MEM_Agg ); + testcase( p->flags & MEM_Dyn ); + testcase( p->flags & MEM_Frame ); + testcase( p->flags & MEM_RowSet ); if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ sqlite3VdbeMemRelease(p); }else if( p->zMalloc ){ @@ -62526,7 +63196,7 @@ static void releaseMemArray(Mem *p, int N){ p->zMalloc = 0; } - p->flags = MEM_Invalid; + p->flags = MEM_Undefined; } db->mallocFailed = malloc_failed; } @@ -62648,7 +63318,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( } if( p->explain==1 ){ pMem->flags = MEM_Int; - pMem->type = SQLITE_INTEGER; pMem->u.i = i; /* Program counter */ pMem++; @@ -62656,7 +63325,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ assert( pMem->z!=0 ); pMem->n = sqlite3Strlen30(pMem->z); - pMem->type = SQLITE_TEXT; pMem->enc = SQLITE_UTF8; pMem++; @@ -62682,24 +63350,21 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->flags = MEM_Int; pMem->u.i = pOp->p1; /* P1 */ - pMem->type = SQLITE_INTEGER; pMem++; pMem->flags = MEM_Int; pMem->u.i = pOp->p2; /* P2 */ - pMem->type = SQLITE_INTEGER; pMem++; pMem->flags = MEM_Int; pMem->u.i = pOp->p3; /* P3 */ - pMem->type = SQLITE_INTEGER; pMem++; if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; zP4 = displayP4(pOp, pMem->z, 32); if( zP4!=pMem->z ){ sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); @@ -62708,7 +63373,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; } - pMem->type = SQLITE_TEXT; pMem++; if( p->explain==1 ){ @@ -62716,10 +63380,9 @@ SQLITE_PRIVATE int sqlite3VdbeList( assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; pMem->n = 2; sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ - pMem->type = SQLITE_TEXT; pMem->enc = SQLITE_UTF8; pMem++; @@ -62728,13 +63391,11 @@ SQLITE_PRIVATE int sqlite3VdbeList( assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; pMem->n = displayComment(pOp, zP4, pMem->z, 500); - pMem->type = SQLITE_TEXT; pMem->enc = SQLITE_UTF8; #else pMem->flags = MEM_Null; /* Comment */ - pMem->type = SQLITE_NULL; #endif } @@ -62757,7 +63418,7 @@ SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe *p){ z = p->zSql; }else if( p->nOp>=1 ){ const VdbeOp *pOp = &p->aOp[0]; - if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ z = pOp->p4.z; while( sqlite3Isspace(*z) ) z++; } @@ -62776,7 +63437,7 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ if( sqlite3IoTrace==0 ) return; if( nOp<1 ) return; pOp = &p->aOp[0]; - if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ int i, j; char z[1000]; sqlite3_snprintf(sizeof(z), z, "%s", pOp->p4.z); @@ -62994,7 +63655,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->aMem--; /* aMem[] goes from 1..nMem */ p->nMem = nMem; /* not from 0..nMem-1 */ for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Invalid; + p->aMem[n].flags = MEM_Undefined; p->aMem[n].db = db; } } @@ -63106,7 +63767,7 @@ static void Cleanup(Vdbe *p){ int i; if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); if( p->aMem ){ - for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Invalid ); + for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); } #endif @@ -63431,7 +64092,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ int nRead = 0; p = db->pVdbe; while( p ){ - if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){ + if( sqlite3_stmt_busy((sqlite3_stmt*)p) ){ cnt++; if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; @@ -63855,12 +64516,24 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ fprintf(out, "%02x", p->aOp[i].opcode); } fprintf(out, "\n"); + if( p->zSql ){ + char c, pc = 0; + fprintf(out, "-- "); + for(i=0; (c = p->zSql[i])!=0; i++){ + if( pc=='\n' ) fprintf(out, "-- "); + putc(c, out); + pc = c; + } + if( pc!='\n' ) fprintf(out, "\n"); + } for(i=0; inOp; i++){ - fprintf(out, "%6d %10lld %8lld ", + char zHdr[100]; + sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", p->aOp[i].cnt, p->aOp[i].cycles, p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 ); + fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); } fclose(out); @@ -64011,7 +64684,7 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( rc ) return rc; if( hasMoved ){ p->cacheStatus = CACHE_STALE; - p->nullRow = 1; + if( hasMoved==2 ) p->nullRow = 1; } } return SQLITE_OK; @@ -64215,6 +64888,14 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ return 0; } +/* Input "x" is a sequence of unsigned characters that represent a +** big-endian integer. Return the equivalent native integer +*/ +#define ONE_BYTE_INT(x) ((i8)(x)[0]) +#define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) +#define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) +#define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + /* ** Deserialize the data blob pointed to by buf as serial type serial_type ** and store the result in pMem. Return the number of bytes read. @@ -64226,7 +64907,6 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( ){ u64 x; u32 y; - int i; switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ @@ -64235,34 +64915,34 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( break; } case 1: { /* 1-byte signed integer */ - pMem->u.i = (signed char)buf[0]; + pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ - i = 256*(signed char)buf[0] | buf[1]; - pMem->u.i = (i64)i; + pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ - i = 65536*(signed char)buf[0] | (buf[1]<<8) | buf[2]; - pMem->u.i = (i64)i; + pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ - y = ((unsigned)buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + y = FOUR_BYTE_UINT(buf); pMem->u.i = (i64)*(int*)&y; pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ - x = 256*(signed char)buf[0] + buf[1]; - y = ((unsigned)buf[2]<<24) | (buf[3]<<16) | (buf[4]<<8) | buf[5]; - x = (x<<32) | y; - pMem->u.i = *(i64*)&x; + pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 6; } case 6: /* 8-byte signed integer */ @@ -64279,12 +64959,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( swapMixedEndianFloat(t2); assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); #endif - x = ((unsigned)buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; - y = ((unsigned)buf[4]<<24) | (buf[5]<<16) | (buf[6]<<8) | buf[7]; + x = FOUR_BYTE_UINT(buf); + y = FOUR_BYTE_UINT(buf+4); x = (x<<32) | y; if( serial_type==6 ){ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); }else{ assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); swapMixedEndianFloat(x); @@ -64376,7 +65057,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( u32 szHdr; Mem *pMem = p->aMem; - p->flags = 0; + p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); idx = getVarint32(aKey, szHdr); d = szHdr; @@ -64397,26 +65078,18 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( p->nField = u; } +#if SQLITE_DEBUG /* -** This function compares the two table rows or index records -** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero -** or positive integer if key1 is less than, equal to or -** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 -** key must be a parsed key such as obtained from -** sqlite3VdbeParseRecord. -** -** Key1 and Key2 do not have to contain the same number of fields. -** The key with fewer fields is usually compares less than the -** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set -** and the common prefixes are equal, then key1 is less than key2. -** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are -** equal, then the keys are considered to be equal and -** the parts beyond the common prefix are ignored. +** This function compares two index or table record keys in the same way +** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), +** this function deserializes and compares values using the +** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used +** in assert() statements to ensure that the optimized code in +** sqlite3VdbeRecordCompare() returns results with these two primitives. */ -SQLITE_PRIVATE int sqlite3VdbeRecordCompare( +static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2 /* Right key */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -64490,24 +65163,594 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( assert( mem1.zMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. If the UNPACKED_INCRKEY - ** flag is set, then break the tie by treating key2 as larger. - ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes - ** are considered to be equal. Otherwise, the longer key is the - ** larger. As it happens, the pPKey2 will always be the longer - ** if there is a difference. - */ - assert( rc==0 ); - if( pPKey2->flags & UNPACKED_INCRKEY ){ - rc = -1; - }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ - /* Leave rc==0 */ - }else if( idx1default_rc; +} +#endif + +/* +** Both *pMem1 and *pMem2 contain string values. Compare the two values +** using the collation sequence pColl. As usual, return a negative , zero +** or positive value if *pMem1 is less than, equal to or greater than +** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". +*/ +static int vdbeCompareMemString( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl +){ + if( pMem1->enc==pColl->enc ){ + /* The strings are already in the correct encoding. Call the + ** comparison function directly */ + return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); + }else{ + int rc; + const void *v1, *v2; + int n1, n2; + Mem c1; + Mem c2; + memset(&c1, 0, sizeof(c1)); + memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + n1 = v1==0 ? 0 : c1.n; + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + n2 = v2==0 ? 0 : c2.n; + rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + sqlite3VdbeMemRelease(&c1); + sqlite3VdbeMemRelease(&c2); + return rc; } - return rc; } + +/* +** Compare the values contained by the two memory cells, returning +** negative, zero or positive if pMem1 is less than, equal to, or greater +** than pMem2. Sorting order is NULL's first, followed by numbers (integers +** and reals) sorted numerically, followed by text ordered by the collating +** sequence pColl and finally blob's ordered by memcmp(). +** +** Two NULL values are considered equal by this function. +*/ +SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ + int rc; + int f1, f2; + int combined_flags; + + f1 = pMem1->flags; + f2 = pMem2->flags; + combined_flags = f1|f2; + assert( (combined_flags & MEM_RowSet)==0 ); + /* If one value is NULL, it is less than the other. If both values + ** are NULL, return 0. + */ + if( combined_flags&MEM_Null ){ + return (f2&MEM_Null) - (f1&MEM_Null); + } + + /* If one value is a number and the other is not, the number is less. + ** If both are numbers, compare as reals if one is a real, or as integers + ** if both values are integers. + */ + if( combined_flags&(MEM_Int|MEM_Real) ){ + double r1, r2; + if( (f1 & f2 & MEM_Int)!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return 1; + return 0; + } + if( (f1&MEM_Real)!=0 ){ + r1 = pMem1->r; + }else if( (f1&MEM_Int)!=0 ){ + r1 = (double)pMem1->u.i; + }else{ + return 1; + } + if( (f2&MEM_Real)!=0 ){ + r2 = pMem2->r; + }else if( (f2&MEM_Int)!=0 ){ + r2 = (double)pMem2->u.i; + }else{ + return -1; + } + if( r1r2 ) return 1; + return 0; + } + + /* If one value is a string and the other is a blob, the string is less. + ** If both are strings, compare using the collating functions. + */ + if( combined_flags&MEM_Str ){ + if( (f1 & MEM_Str)==0 ){ + return 1; + } + if( (f2 & MEM_Str)==0 ){ + return -1; + } + + assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==SQLITE_UTF8 || + pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); + + /* The collation sequence must be defined at this point, even if + ** the user deletes the collation sequence after the vdbe program is + ** compiled (this was not always the case). + */ + assert( !pColl || pColl->xCmp ); + + if( pColl ){ + return vdbeCompareMemString(pMem1, pMem2, pColl); + } + /* If a NULL pointer was passed as the collate function, fall through + ** to the blob case and use memcmp(). */ + } + + /* Both values must be blobs. Compare using memcmp(). */ + rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); + if( rc==0 ){ + rc = pMem1->n - pMem2->n; + } + return rc; +} + + +/* +** The first argument passed to this function is a serial-type that +** corresponds to an integer - all values between 1 and 9 inclusive +** except 7. The second points to a buffer containing an integer value +** serialized according to serial_type. This function deserializes +** and returns the value. +*/ +static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + u32 y; + assert( CORRUPT_DB || (serial_type>=1 && serial_type<=9 && serial_type!=7) ); + switch( serial_type ){ + case 0: + case 1: + testcase( aKey[0]&0x80 ); + return ONE_BYTE_INT(aKey); + case 2: + testcase( aKey[0]&0x80 ); + return TWO_BYTE_INT(aKey); + case 3: + testcase( aKey[0]&0x80 ); + return THREE_BYTE_INT(aKey); + case 4: { + testcase( aKey[0]&0x80 ); + y = FOUR_BYTE_UINT(aKey); + return (i64)*(int*)&y; + } + case 5: { + testcase( aKey[0]&0x80 ); + return FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + } + case 6: { + u64 x = FOUR_BYTE_UINT(aKey); + testcase( aKey[0]&0x80 ); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + return (i64)*(i64*)&x; + } + } + + return (serial_type - 8); +} + +/* +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. +** +** If argument bSkip is non-zero, it is assumed that the caller has already +** determined that the first fields of the keys are equal. +** +** Key1 and Key2 do not have to contain the same number of fields. If all +** fields that appear in both keys are equal, then pPKey2->default_rc is +** returned. +** +** If database corruption is discovered, set pPKey2->isCorrupt to non-zero +** and return 0. +*/ +SQLITE_PRIVATE int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* If true, skip the first field */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + int i; /* Index of next field to compare */ + u32 szHdr1; /* Size of record header in bytes */ + u32 idx1; /* Offset of first type in header */ + int rc = 0; /* Return value */ + Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ + KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + Mem mem1; + + /* If bSkip is true, then the caller has already determined that the first + ** two elements in the keys are equal. Fix the various stack variables so + ** that this routine begins comparing at the second field. */ + if( bSkip ){ + u32 s1; + idx1 = 1 + getVarint32(&aKey1[1], s1); + szHdr1 = aKey1[0]; + d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); + i = 1; + pRhs++; + }else{ + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + if( d1>(unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + i = 0; + } + + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField + || CORRUPT_DB ); + assert( pPKey2->pKeyInfo->aSortOrder!=0 ); + assert( pPKey2->pKeyInfo->nField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type; + + /* RHS is an integer */ + if( pRhs->flags & MEM_Int ){ + serial_type = aKey1[idx1]; + testcase( serial_type==12 ); + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else if( serial_type==7 ){ + double rhs = (double)pRhs->u.i; + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( mem1.rrhs ){ + rc = +1; + } + }else{ + i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); + i64 rhs = pRhs->u.i; + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is real */ + else if( pRhs->flags & MEM_Real ){ + serial_type = aKey1[idx1]; + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else{ + double rhs = pRhs->r; + double lhs; + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( serial_type==7 ){ + lhs = mem1.r; + }else{ + lhs = (double)mem1.u.i; + } + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is a string */ + else if( pRhs->flags & MEM_Str ){ + getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 ){ + rc = -1; + }else if( !(serial_type & 0x01) ){ + rc = +1; + }else{ + mem1.n = (serial_type - 12) / 2; + testcase( (d1+mem1.n)==(unsigned)nKey1 ); + testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); + if( (d1+mem1.n) > (unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else if( pKeyInfo->aColl[i] ){ + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + mem1.flags = MEM_Str; + mem1.z = (char*)&aKey1[d1]; + rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + }else{ + int nCmp = MIN(mem1.n, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = mem1.n - pRhs->n; + } + } + } + + /* RHS is a blob */ + else if( pRhs->flags & MEM_Blob ){ + getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 || (serial_type & 0x01) ){ + rc = -1; + }else{ + int nStr = (serial_type - 12) / 2; + testcase( (d1+nStr)==(unsigned)nKey1 ); + testcase( (d1+nStr+1)==(unsigned)nKey1 ); + if( (d1+nStr) > (unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else{ + int nCmp = MIN(nStr, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = nStr - pRhs->n; + } + } + } + + /* RHS is null */ + else{ + serial_type = aKey1[idx1]; + rc = (serial_type!=0); + } + + if( rc!=0 ){ + if( pKeyInfo->aSortOrder[i] ){ + rc = -rc; + } + assert( CORRUPT_DB + || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || pKeyInfo->db->mallocFailed + ); + assert( mem1.zMalloc==0 ); /* See comment below */ + return rc; + } + + i++; + pRhs++; + d1 += sqlite3VdbeSerialTypeLen(serial_type); + idx1 += sqlite3VarintLen(serial_type); + }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ + assert( mem1.zMalloc==0 ); + + /* rc==0 here means that one or both of the keys ran out of fields and + ** all the fields up to that point were equal. Return the the default_rc + ** value. */ + assert( CORRUPT_DB + || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || pKeyInfo->db->mallocFailed + ); + return pPKey2->default_rc; +} + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is an integer, and (b) the +** size-of-header varint at the start of (pKey1/nKey1) fits in a single +** byte (i.e. is less than 128). +** +** To avoid concerns about buffer overreads, this routine is only used +** on schemas where the maximum valid header size is 63 bytes or less. +*/ +static int vdbeRecordCompareInt( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* Ignored */ +){ + const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; + int serial_type = ((const u8*)pKey1)[1]; + int res; + u32 y; + u64 x; + i64 v = pPKey2->aMem[0].u.i; + i64 lhs; + UNUSED_PARAMETER(bSkip); + + assert( bSkip==0 ); + assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); + switch( serial_type ){ + case 1: { /* 1-byte signed integer */ + lhs = ONE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 2: { /* 2-byte signed integer */ + lhs = TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 3: { /* 3-byte signed integer */ + lhs = THREE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 4: { /* 4-byte signed integer */ + y = FOUR_BYTE_UINT(aKey); + lhs = (i64)*(int*)&y; + testcase( lhs<0 ); + break; + } + case 5: { /* 6-byte signed integer */ + lhs = FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 6: { /* 8-byte signed integer */ + x = FOUR_BYTE_UINT(aKey); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + lhs = *(i64*)&x; + testcase( lhs<0 ); + break; + } + case 8: + lhs = 0; + break; + case 9: + lhs = 1; + break; + + /* This case could be removed without changing the results of running + ** this code. Including it causes gcc to generate a faster switch + ** statement (since the range of switch targets now starts at zero and + ** is contiguous) but does not cause any duplicate code to be generated + ** (as gcc is clever enough to combine the two like cases). Other + ** compilers might be similar. */ + case 0: case 7: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + + default: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + } + + if( v>lhs ){ + res = pPKey2->r1; + }else if( vr2; + }else if( pPKey2->nField>1 ){ + /* The first fields of the two keys are equal. Compare the trailing + ** fields. */ + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + }else{ + /* The first fields of the two keys are equal and there are no trailing + ** fields. Return pPKey2->default_rc in this case. */ + res = pPKey2->default_rc; + } + + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + ); + return res; +} + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is a string, that (b) the first field +** uses the collation sequence BINARY and (c) that the size-of-header varint +** at the start of (pKey1/nKey1) fits in a single byte. +*/ +static int vdbeRecordCompareString( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip +){ + const u8 *aKey1 = (const u8*)pKey1; + int serial_type; + int res; + UNUSED_PARAMETER(bSkip); + + assert( bSkip==0 ); + getVarint32(&aKey1[1], serial_type); + + if( serial_type<12 ){ + res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ + res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ + }else{ + int nCmp; + int nStr; + int szHdr = aKey1[0]; + + nStr = (serial_type-12) / 2; + if( (szHdr + nStr) > nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + + if( res==0 ){ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ){ + if( pPKey2->nField>1 ){ + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + }else{ + res = pPKey2->default_rc; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + } + + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + || pPKey2->pKeyInfo->db->mallocFailed + ); + return res; +} + +/* +** Return a pointer to an sqlite3VdbeRecordCompare() compatible function +** suitable for comparing serialized records to the unpacked record passed +** as the only argument. +*/ +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ + /* varintRecordCompareInt() and varintRecordCompareString() both assume + ** that the size-of-header varint that occurs at the start of each record + ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt() + ** also assumes that it is safe to overread a buffer by at least the + ** maximum possible legal header size plus 8 bytes. Because there is + ** guaranteed to be at least 74 (but not 136) bytes of padding following each + ** buffer passed to varintRecordCompareInt() this makes it convenient to + ** limit the size of the header to 64 bytes in cases where the first field + ** is an integer. + ** + ** The easiest way to enforce this limit is to consider only records with + ** 13 fields or less. If the first field is an integer, the maximum legal + ** header size is (12*5 + 1 + 1) bytes. */ + if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=13 ){ + int flags = p->aMem[0].flags; + if( p->pKeyInfo->aSortOrder[0] ){ + p->r1 = 1; + p->r2 = -1; + }else{ + p->r1 = -1; + p->r2 = 1; + } + if( (flags & MEM_Int) ){ + return vdbeRecordCompareInt; + } + testcase( flags & MEM_Real ); + testcase( flags & MEM_Null ); + testcase( flags & MEM_Blob ); + if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + assert( flags & MEM_Str ); + return vdbeRecordCompareString; + } + } + + return sqlite3VdbeRecordCompare; +} /* ** pCur points at an index entry created using the OP_MakeRecord opcode. @@ -64598,9 +65841,9 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ ** of the keys prior to the final rowid, not the entire key. */ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( - VdbeCursor *pC, /* The cursor to compare against */ - UnpackedRecord *pUnpacked, /* Unpacked version of key to compare against */ - int *res /* Write the comparison result here */ + VdbeCursor *pC, /* The cursor to compare against */ + UnpackedRecord *pUnpacked, /* Unpacked version of key */ + int *res /* Write the comparison result here */ ){ i64 nCellKey = 0; int rc; @@ -64610,7 +65853,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( assert( sqlite3BtreeCursorIsValid(pCur) ); VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ - /* nCellKey will always be between 0 and 0xffffffff because of the say + /* nCellKey will always be between 0 and 0xffffffff because of the way ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ *res = 0; @@ -64621,8 +65864,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - assert( pUnpacked->flags & UNPACKED_PREFIX_MATCH ); - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } @@ -64686,7 +65928,6 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff if( pRet ){ sqlite3VdbeMemCopy((Mem *)pRet, pMem); sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); - sqlite3VdbeMemStoreType((Mem *)pRet); } return pRet; } @@ -64860,7 +66101,6 @@ SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ sqlite3VdbeMemExpandBlob(p); - p->flags &= ~MEM_Str; p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ @@ -64897,7 +66137,41 @@ SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ } #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ - return pVal->type; + static const u8 aType[] = { + SQLITE_BLOB, /* 0x00 */ + SQLITE_NULL, /* 0x01 */ + SQLITE_TEXT, /* 0x02 */ + SQLITE_NULL, /* 0x03 */ + SQLITE_INTEGER, /* 0x04 */ + SQLITE_NULL, /* 0x05 */ + SQLITE_INTEGER, /* 0x06 */ + SQLITE_NULL, /* 0x07 */ + SQLITE_FLOAT, /* 0x08 */ + SQLITE_NULL, /* 0x09 */ + SQLITE_FLOAT, /* 0x0a */ + SQLITE_NULL, /* 0x0b */ + SQLITE_INTEGER, /* 0x0c */ + SQLITE_NULL, /* 0x0d */ + SQLITE_INTEGER, /* 0x0e */ + SQLITE_NULL, /* 0x0f */ + SQLITE_BLOB, /* 0x10 */ + SQLITE_NULL, /* 0x11 */ + SQLITE_TEXT, /* 0x12 */ + SQLITE_NULL, /* 0x13 */ + SQLITE_INTEGER, /* 0x14 */ + SQLITE_NULL, /* 0x15 */ + SQLITE_INTEGER, /* 0x16 */ + SQLITE_NULL, /* 0x17 */ + SQLITE_FLOAT, /* 0x18 */ + SQLITE_NULL, /* 0x19 */ + SQLITE_FLOAT, /* 0x1a */ + SQLITE_NULL, /* 0x1b */ + SQLITE_INTEGER, /* 0x1c */ + SQLITE_NULL, /* 0x1d */ + SQLITE_INTEGER, /* 0x1e */ + SQLITE_NULL, /* 0x1f */ + }; + return aType[pVal->flags&MEM_AffMask]; } /**************************** sqlite3_result_ ******************************* @@ -65418,6 +66692,30 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ return pVm->nResColumn; } +/* +** Return a pointer to static memory containing an SQL NULL value. +*/ +static const Mem *columnNullValue(void){ + /* Even though the Mem structure contains an element + ** of type i64, on certain architectures (x86) with certain compiler + ** switches (-Os), gcc may align this Mem object on a 4-byte boundary + ** instead of an 8-byte one. This all works fine, except that when + ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s + ** that a Mem structure is located on an 8-byte boundary. To prevent + ** these assert()s from failing, when building with SQLITE_DEBUG defined + ** using gcc, we force nullMem to be 8-byte aligned using the magical + ** __attribute__((aligned(8))) macro. */ + static const Mem nullMem +#if defined(SQLITE_DEBUG) && defined(__GNUC__) + __attribute__((aligned(8))) +#endif + = {0, "", (double)0, {0}, 0, MEM_Null, 0, +#ifdef SQLITE_DEBUG + 0, 0, /* pScopyFrom, pFiller */ +#endif + 0, 0 }; + return &nullMem; +} /* ** Check to see if column iCol of the given statement is valid. If @@ -65434,32 +66732,11 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ sqlite3_mutex_enter(pVm->db->mutex); pOut = &pVm->pResultSet[i]; }else{ - /* If the value passed as the second argument is out of range, return - ** a pointer to the following static Mem object which contains the - ** value SQL NULL. Even though the Mem structure contains an element - ** of type i64, on certain architectures (x86) with certain compiler - ** switches (-Os), gcc may align this Mem object on a 4-byte boundary - ** instead of an 8-byte one. This all works fine, except that when - ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s - ** that a Mem structure is located on an 8-byte boundary. To prevent - ** these assert()s from failing, when building with SQLITE_DEBUG defined - ** using gcc, we force nullMem to be 8-byte aligned using the magical - ** __attribute__((aligned(8))) macro. */ - static const Mem nullMem -#if defined(SQLITE_DEBUG) && defined(__GNUC__) - __attribute__((aligned(8))) -#endif - = {0, "", (double)0, {0}, 0, MEM_Null, SQLITE_NULL, 0, -#ifdef SQLITE_DEBUG - 0, 0, /* pScopyFrom, pFiller */ -#endif - 0, 0 }; - if( pVm && ALWAYS(pVm->db) ){ sqlite3_mutex_enter(pVm->db->mutex); sqlite3Error(pVm->db, SQLITE_RANGE, 0); } - pOut = (Mem*)&nullMem; + pOut = (Mem*)columnNullValue(); } return pOut; } @@ -65856,7 +67133,7 @@ SQLITE_API int sqlite3_bind_text16( #endif /* SQLITE_OMIT_UTF16 */ SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; - switch( pValue->type ){ + switch( sqlite3_value_type((sqlite3_value*)pValue) ){ case SQLITE_INTEGER: { rc = sqlite3_bind_int64(pStmt, i, pValue->u.i); break; @@ -66012,7 +67289,7 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ */ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->pc>0 && v->magic==VDBE_MAGIC_RUN; + return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; } /* @@ -66357,33 +67634,8 @@ SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** The code in this file implements execution method of the -** Virtual Database Engine (VDBE). A separate file ("vdbeaux.c") -** handles housekeeping details such as creating and deleting -** VDBE instances. This file is solely interested in executing -** the VDBE program. -** -** In the external interface, an "sqlite3_stmt*" is an opaque pointer -** to a VDBE. -** -** The SQL parser generates a program which is then executed by -** the VDBE to do the work of the SQL statement. VDBE programs are -** similar in form to assembly language. The program consists of -** a linear sequence of operations. Each operation has an opcode -** and 5 operands. Operands P1, P2, and P3 are integers. Operand P4 -** is a null-terminated string. Operand P5 is an unsigned character. -** Few opcodes use all 5 operands. -** -** Computation results are stored on a set of registers numbered beginning -** with 1 and going up to Vdbe.nMem. Each register can store -** either an integer, a null-terminated string, a floating point -** number, or the SQL "NULL" value. An implicit conversion from one -** type to the other occurs as necessary. -** -** Most of the code in this file is taken up by the sqlite3VdbeExec() -** function which does the work of interpreting a VDBE program. -** But other routines are also provided to help in building up -** a program instruction by instruction. +** The code in this file implements the function that runs the +** bytecode of a prepared statement. ** ** Various scripts scan this source file in order to generate HTML ** documentation, headers files, or other derived files. The formatting @@ -66395,7 +67647,11 @@ SQLITE_PRIVATE const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ /* ** Invoke this macro on memory cells just prior to changing the ** value of the cell. This macro verifies that shallow copies are -** not misused. +** not misused. A shallow copy of a string or blob just copies a +** pointer to the string or blob, not the content. If the original +** is changed while the copy is still in use, the string or blob might +** be changed out from under the copy. This macro verifies that nothing +** like that ever happens. */ #ifdef SQLITE_DEBUG # define memAboutToChange(P,M) sqlite3VdbeMemAboutToChange(P,M) @@ -66454,7 +67710,7 @@ static void updateMaxBlobsize(Mem *p){ #endif /* -** The next global variable is incremented each type the OP_Found opcode +** The next global variable is incremented each time the OP_Found opcode ** is executed. This is used to test whether or not the foreign key ** operation implemented using OP_FkIsZero is working. This variable ** has no function other than to help verify the correct operation of the @@ -66474,6 +67730,34 @@ SQLITE_API int sqlite3_found_count = 0; # define UPDATE_MAX_BLOBSIZE(P) #endif +/* +** Invoke the VDBE coverage callback, if that callback is defined. This +** feature is used for test suite validation only and does not appear an +** production builds. +** +** M is an integer, 2 or 3, that indices how many different ways the +** branch can go. It is usually 2. "I" is the direction the branch +** goes. 0 means falls through. 1 means branch is taken. 2 means the +** second alternative branch is taken. +*/ +#if !defined(SQLITE_VDBE_COVERAGE) +# define VdbeBranchTaken(I,M) +#else +# define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) + static void vdbeTakeBranch(int iSrcLine, u8 I, u8 M){ + if( iSrcLine<=2 && ALWAYS(iSrcLine>0) ){ + M = iSrcLine; + /* Assert the truth of VdbeCoverageAlwaysTaken() and + ** VdbeCoverageNeverTaken() */ + assert( (M & I)==I ); + }else{ + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine,I,M); + } + } +#endif + /* ** Convert the given register into a string if it isn't one ** already. Return non-zero if a malloc() fails. @@ -66491,38 +67775,14 @@ SQLITE_API int sqlite3_found_count = 0; ** ** This routine converts an ephemeral string into a dynamically allocated ** string that the register itself controls. In other words, it -** converts an MEM_Ephem string into an MEM_Dyn string. +** converts an MEM_Ephem string into a string with P.z==P.zMalloc. */ #define Deephemeralize(P) \ if( ((P)->flags&MEM_Ephem)!=0 \ && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -# define isSorter(x) ((x)->pSorter!=0) - -/* -** Argument pMem points at a register that will be passed to a -** user-defined function or returned to the user as the result of a query. -** This routine sets the pMem->type variable used by the sqlite3_value_*() -** routines. -*/ -SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem){ - int flags = pMem->flags; - if( flags & MEM_Null ){ - pMem->type = SQLITE_NULL; - } - else if( flags & MEM_Int ){ - pMem->type = SQLITE_INTEGER; - } - else if( flags & MEM_Real ){ - pMem->type = SQLITE_FLOAT; - } - else if( flags & MEM_Str ){ - pMem->type = SQLITE_TEXT; - }else{ - pMem->type = SQLITE_BLOB; - } -} +#define isSorter(x) ((x)->pSorter!=0) /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL @@ -66652,12 +67912,13 @@ static void applyAffinity( ** loss of information and return the revised type of the argument. */ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ - Mem *pMem = (Mem*)pVal; - if( pMem->type==SQLITE_TEXT ){ + int eType = sqlite3_value_type(pVal); + if( eType==SQLITE_TEXT ){ + Mem *pMem = (Mem*)pVal; applyNumericAffinity(pMem); - sqlite3VdbeMemStoreType(pMem); + eType = sqlite3_value_type(pVal); } - return pMem->type; + return eType; } /* @@ -66672,6 +67933,29 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or +** none. +** +** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. +** But it does set pMem->r and pMem->u.i appropriately. +*/ +static u16 numericType(Mem *pMem){ + if( pMem->flags & (MEM_Int|MEM_Real) ){ + return pMem->flags & (MEM_Int|MEM_Real); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ + if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; + } + return 0; +} + #ifdef SQLITE_DEBUG /* ** Write a nice string representation of the contents of cell pMem @@ -66760,7 +68044,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ ** Print the value of a register for tracing purposes: */ static void memTracePrint(Mem *p){ - if( p->flags & MEM_Invalid ){ + if( p->flags & MEM_Undefined ){ printf(" undefined"); }else if( p->flags & MEM_Null ){ printf(" NULL"); @@ -66893,20 +68177,6 @@ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } #endif -/* -** The CHECK_FOR_INTERRUPT macro defined here looks to see if the -** sqlite3_interrupt() routine has been called. If it has been, then -** processing of the VDBE program is interrupted. -** -** This macro added to every instruction that does a jump in order to -** implement a loop. This test used to be on every single instruction, -** but that meant we more testing than we needed. By only testing the -** flag on jump instructions, we get a (small) speed improvement. -*/ -#define CHECK_FOR_INTERRUPT \ - if( db->u1.isInterrupted ) goto abort_due_to_interrupt; - - #ifndef NDEBUG /* ** This function is only called from within an assert() expression. It @@ -66929,35 +68199,8 @@ static int checkSavepointCount(sqlite3 *db){ /* -** Execute as much of a VDBE program as we can then return. -** -** sqlite3VdbeMakeReady() must be called before this routine in order to -** close the program with a final OP_Halt and to set up the callbacks -** and the error message pointer. -** -** Whenever a row or result data is available, this routine will either -** invoke the result callback (if there is one) or return with -** SQLITE_ROW. -** -** If an attempt is made to open a locked database, then this routine -** will either invoke the busy callback (if there is one) or it will -** return SQLITE_BUSY. -** -** If an error occurs, an error message is written to memory obtained -** from sqlite3_malloc() and p->zErrMsg is made to point to that memory. -** The error code is stored in p->rc and this routine returns SQLITE_ERROR. -** -** If the callback ever returns non-zero, then the program exits -** immediately. There will be no error message but the p->rc field is -** set to SQLITE_ABORT and this routine will return SQLITE_ERROR. -** -** A memory allocation error causes p->rc to be set to SQLITE_NOMEM and this -** routine to return SQLITE_ERROR. -** -** Other fatal errors return SQLITE_ERROR. -** -** After this routine has finished, sqlite3VdbeFinalize() should be -** used to clean up the mess that was left behind. +** Execute as much of a VDBE program as we can. +** This is the core of sqlite3_step(). */ SQLITE_PRIVATE int sqlite3VdbeExec( Vdbe *p /* The VDBE */ @@ -66983,7 +68226,6 @@ SQLITE_PRIVATE int sqlite3VdbeExec( i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ - int origPc; /* Program counter at start of opcode */ #endif /*** INSERT STACK UNION HERE ***/ @@ -67001,7 +68243,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( p->explain==0 ); p->pResultSet = 0; db->busyHandler.nBusy = 0; - CHECK_FOR_INTERRUPT; + if( db->u1.isInterrupted ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ @@ -67045,7 +68287,6 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pc>=0 && pcnOp ); if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE - origPc = pc; start = sqlite3Hwtime(); #endif nVmStep++; @@ -67093,18 +68334,21 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pOp->p1>0 ); assert( pOp->p1<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ @@ -67162,6 +68406,11 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** The next instruction executed will be ** the one at index P2 from the beginning of ** the program. +** +** The P1 parameter is not actually used by this opcode. However, it +** is sometimes set to 1 instead of 0 as a hint to the command-line shell +** that this Goto is the bottom of a loop and that the lines from P2 down +** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ pc = pOp->p2 - 1; @@ -67177,7 +68426,7 @@ case OP_Goto: { /* jump */ ** checks on every opcode. This helps sqlite3_step() to run about 1.5% ** faster according to "valgrind --tool=cachegrind" */ check_for_interrupt: - CHECK_FOR_INTERRUPT; + if( db->u1.isInterrupted ) goto abort_due_to_interrupt; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Call the progress callback if it is configured and the required number ** of VDBE ops have been executed (either since this invocation of @@ -67206,7 +68455,7 @@ case OP_Goto: { /* jump */ case OP_Gosub: { /* jump */ assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); + assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; pIn1->u.i = pc; @@ -67217,23 +68466,69 @@ case OP_Gosub: { /* jump */ /* Opcode: Return P1 * * * * ** -** Jump to the next instruction after the address in register P1. +** Jump to the next instruction after the address in register P1. After +** the jump, register P1 becomes undefined. */ case OP_Return: { /* in1 */ pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags & MEM_Int ); + assert( pIn1->flags==MEM_Int ); pc = (int)pIn1->u.i; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: InitCoroutine P1 P2 P3 * * +** +** Set up register P1 so that it will OP_Yield to the co-routine +** located at address P3. +** +** If P2!=0 then the co-routine implementation immediately follows +** this opcode. So jump over the co-routine implementation to +** address P2. +*/ +case OP_InitCoroutine: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p2>=0 && pOp->p2nOp ); + assert( pOp->p3>=0 && pOp->p3nOp ); + pOut = &aMem[pOp->p1]; + assert( !VdbeMemDynamic(pOut) ); + pOut->u.i = pOp->p3 - 1; + pOut->flags = MEM_Int; + if( pOp->p2 ) pc = pOp->p2 - 1; break; } -/* Opcode: Yield P1 * * * * +/* Opcode: EndCoroutine P1 * * * * +** +** The instruction at the address in register P1 is an OP_Yield. +** Jump to the P2 parameter of that OP_Yield. +** After the jump, register P1 becomes undefined. +*/ +case OP_EndCoroutine: { /* in1 */ + VdbeOp *pCaller; + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags==MEM_Int ); + assert( pIn1->u.i>=0 && pIn1->u.inOp ); + pCaller = &aOp[pIn1->u.i]; + assert( pCaller->opcode==OP_Yield ); + assert( pCaller->p2>=0 && pCaller->p2nOp ); + pc = pCaller->p2 - 1; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: Yield P1 P2 * * * ** ** Swap the program counter with the value in register P1. +** +** If the co-routine ends with OP_Yield or OP_Return then continue +** to the next instruction. But if the co-routine ends with +** OP_EndCoroutine, jump immediately to P2. */ -case OP_Yield: { /* in1 */ +case OP_Yield: { /* in1, jump */ int pcDest; pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); + assert( VdbeMemDynamic(pIn1)==0 ); pIn1->flags = MEM_Int; pcDest = (int)pIn1->u.i; pIn1->u.i = pc; @@ -67243,7 +68538,7 @@ case OP_Yield: { /* in1 */ } /* Opcode: HaltIfNull P1 P2 P3 P4 P5 -** Synopsis: if r[P3] null then halt +** Synopsis: if r[P3]=null halt ** ** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the @@ -67391,7 +68686,9 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ ** Synopsis: r[P2]='P4' ** ** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into an OP_String before it is executed for the first time. +** into an OP_String before it is executed for the first time. During +** this transformation, the length of string P4 is computed and stored +** as the P1 parameter. */ case OP_String8: { /* same as TK_STRING, out2-prerelease */ assert( pOp->p4.z!=0 ); @@ -67404,10 +68701,9 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ if( rc==SQLITE_TOOBIG ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->zMalloc==pOut->z ); - assert( pOut->flags & MEM_Dyn ); + assert( VdbeMemDynamic(pOut)==0 ); pOut->zMalloc = 0; pOut->flags |= MEM_Static; - pOut->flags &= ~MEM_Dyn; if( pOp->p4type==P4_DYNAMIC ){ sqlite3DbFree(db, pOp->p4.z); } @@ -67465,8 +68761,22 @@ case OP_Null: { /* out2-prerelease */ break; } +/* Opcode: SoftNull P1 * * * * +** Synopsis: r[P1]=NULL +** +** Set register P1 to have the value NULL as seen by the OP_MakeRecord +** instruction, but do not free any string or blob memory associated with +** the register, so that if the value was a string or blob that was +** previously copied using OP_SCopy, the copies will continue to be valid. +*/ +case OP_SoftNull: { + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + pOut = &aMem[pOp->p1]; + pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; + break; +} -/* Opcode: Blob P1 P2 * P4 +/* Opcode: Blob P1 P2 * P4 * ** Synopsis: r[P2]=P4 (len=P1) ** ** P4 points to a blob of data P1 bytes long. Store this @@ -67485,7 +68795,7 @@ case OP_Blob: { /* out2-prerelease */ ** ** Transfer the values of bound parameter P1 into register P2 ** -** If the parameter is named, then its name appears in P4 and P3==1. +** If the parameter is named, then its name appears in P4. ** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2-prerelease */ @@ -67505,10 +68815,11 @@ case OP_Variable: { /* out2-prerelease */ /* Opcode: Move P1 P2 P3 * * ** Synopsis: r[P2@P3]=r[P1@P3] ** -** Move the values in register P1..P1+P3 over into -** registers P2..P2+P3. Registers P1..P1+P3 are +** Move the P3 values in register P1..P1+P3-1 over into +** registers P2..P2+P3-1. Registers P1..P1+P3-1 are ** left holding a NULL. It is an error for register ranges -** P1..P1+P3 and P2..P2+P3 to overlap. +** P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error +** for P3 to be less than 1. */ case OP_Move: { char *zMalloc; /* Holding variable for allocated memory */ @@ -67519,7 +68830,7 @@ case OP_Move: { n = pOp->p3; p1 = pOp->p1; p2 = pOp->p2; - assert( n>=0 && p1>0 && p2>0 ); + assert( n>0 && p1>0 && p2>0 ); assert( p1+n<=p2 || p2+n<=p1 ); pIn1 = &aMem[p1]; @@ -67529,19 +68840,21 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); + VdbeMemRelease(pOut); zMalloc = pOut->zMalloc; - pOut->zMalloc = 0; - sqlite3VdbeMemMove(pOut, pIn1); + memcpy(pOut, pIn1, sizeof(Mem)); #ifdef SQLITE_DEBUG if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ pOut->pScopyFrom += p1 - pOp->p2; } #endif + pIn1->flags = MEM_Undefined; + pIn1->xDel = 0; pIn1->zMalloc = zMalloc; REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; - }while( n-- ); + }while( --n ); break; } @@ -67604,8 +68917,8 @@ case OP_SCopy: { /* out2 */ ** The registers P1 through P1+P2-1 contain a single row of ** results. This opcode causes the sqlite3_step() call to terminate ** with an SQLITE_ROW return code and it sets up the sqlite3_stmt -** structure to provide access to the top P1 values as the result -** row. +** structure to provide access to the r(P1)..r(P1+P2-1) values as +** the result row. */ case OP_ResultRow: { Mem *pMem; @@ -67670,7 +68983,6 @@ case OP_ResultRow: { assert( (pMem[i].flags & MEM_Ephem)==0 || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); sqlite3VdbeMemNulTerminate(&pMem[i]); - sqlite3VdbeMemStoreType(&pMem[i]); REGISTER_TRACE(pOp->p1+i, &pMem[i]); } if( db->mallocFailed ) goto no_mem; @@ -67713,10 +69025,10 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - MemSetTypeFlag(pOut, MEM_Str); if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } + MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); } @@ -67774,20 +69086,22 @@ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ char bIntint; /* Started out as two integer operands */ - int flags; /* Combined MEM_* flags from both inputs */ + u16 flags; /* Combined MEM_* flags from both inputs */ + u16 type1; /* Numeric type of left operand */ + u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ i64 iB; /* Integer value of right operand */ double rA; /* Real value of left operand */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - applyNumericAffinity(pIn1); + type1 = numericType(pIn1); pIn2 = &aMem[pOp->p2]; - applyNumericAffinity(pIn2); + type2 = numericType(pIn2); pOut = &aMem[pOp->p3]; flags = pIn1->flags | pIn2->flags; if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; - if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ + if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; bIntint = 1; @@ -67843,7 +69157,7 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( (flags & MEM_Real)==0 && !bIntint ){ + if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); } #endif @@ -67916,7 +69230,6 @@ case OP_Function: { assert( memIsValid(pArg) ); apVal[i] = pArg; Deephemeralize(pArg); - sqlite3VdbeMemStoreType(pArg); REGISTER_TRACE(pOp->p2+i, pArg); } @@ -68095,6 +69408,7 @@ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); if( (pIn1->flags & MEM_Int)==0 ){ if( pOp->p2==0 ){ rc = SQLITE_MISMATCH; @@ -68133,7 +69447,7 @@ case OP_RealAffinity: { /* in1 */ ** ** Force the value in register P1 to be text. ** If the value is numeric, convert it to a string using the -** equivalent of printf(). Blob values are unchanged and +** equivalent of sprintf(). Blob values are unchanged and ** are afterwards simply interpreted as text. ** ** A NULL value is not changed by this routine. It remains NULL. @@ -68335,6 +69649,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); assert( (flags1 & MEM_Cleared)==0 ); + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 ); if( (flags1&MEM_Null)!=0 && (flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 @@ -68348,12 +69663,15 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** then the result is always NULL. ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. */ - if( pOp->p5 & SQLITE_JUMPIFNULL ){ - pc = pOp->p2-1; - }else if( pOp->p5 & SQLITE_STOREP2 ){ + if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; MemSetTypeFlag(pOut, MEM_Null); REGISTER_TRACE(pOp->p2, pOut); + }else{ + VdbeBranchTaken(2,3); + if( pOp->p5 & SQLITE_JUMPIFNULL ){ + pc = pOp->p2-1; + } } break; } @@ -68386,10 +69704,12 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = res; REGISTER_TRACE(pOp->p2, pOut); - }else if( res ){ - pc = pOp->p2-1; + }else{ + VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + if( res ){ + pc = pOp->p2-1; + } } - /* Undo any changes made by applyAffinity() to the input registers. */ pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); @@ -68413,6 +69733,7 @@ case OP_Permutation: { } /* Opcode: Compare P1 P2 P3 P4 P5 +** Synopsis: r[P1@P3] <-> r[P2@P3] ** ** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this ** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of @@ -68486,11 +69807,11 @@ case OP_Compare: { */ case OP_Jump: { /* jump */ if( iCompare<0 ){ - pc = pOp->p1 - 1; + pc = pOp->p1 - 1; VdbeBranchTaken(0,3); }else if( iCompare==0 ){ - pc = pOp->p2 - 1; + pc = pOp->p2 - 1; VdbeBranchTaken(1,3); }else{ - pc = pOp->p3 - 1; + pc = pOp->p3 - 1; VdbeBranchTaken(2,3); } break; } @@ -68588,10 +69909,13 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: Once P1 P2 * * * ** ** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, -** set the flag and fall through to the next instruction. +** set the flag and fall through to the next instruction. In other words, +** this opcode causes all following opcodes up through P2 (but not including +** P2) to run just once and to be skipped on subsequent times through the loop. */ case OP_Once: { /* jump */ assert( pOp->p1nOnceFlag ); + VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); if( p->aOnceFlag[pOp->p1] ){ pc = pOp->p2-1; }else{ @@ -68626,6 +69950,7 @@ case OP_IfNot: { /* jump, in1 */ #endif if( pOp->opcode==OP_IfNot ) c = !c; } + VdbeBranchTaken(c!=0, 2); if( c ){ pc = pOp->p2-1; } @@ -68639,6 +69964,7 @@ case OP_IfNot: { /* jump, in1 */ */ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)!=0, 2); if( (pIn1->flags & MEM_Null)!=0 ){ pc = pOp->p2 - 1; } @@ -68652,6 +69978,7 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ */ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2); if( (pIn1->flags & MEM_Null)==0 ){ pc = pOp->p2 - 1; } @@ -68728,11 +70055,6 @@ case OP_Column: { if( pCrsr==0 ){ assert( pC->pseudoTableReg>0 ); pReg = &aMem[pC->pseudoTableReg]; - if( pC->multiPseudo ){ - sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem); - Deephemeralize(pDest); - goto op_column_out; - } assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); pC->payloadSize = pC->szRow = avail = pReg->n; @@ -68883,6 +70205,7 @@ case OP_Column: { */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); + assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ @@ -68920,8 +70243,8 @@ case OP_Column: { ** This prevents a memory copy. */ if( sMem.zMalloc ){ assert( sMem.z==sMem.zMalloc ); - assert( !(pDest->flags & MEM_Dyn) ); - assert( !(pDest->flags & (MEM_Blob|MEM_Str)) || pDest->z==sMem.z ); + assert( VdbeMemDynamic(pDest)==0 ); + assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); pDest->flags &= ~(MEM_Ephem|MEM_Static); pDest->flags |= MEM_Term; pDest->z = sMem.z; @@ -68958,7 +70281,6 @@ case OP_Affinity: { while( (cAff = *(zAffinity++))!=0 ){ assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); - ExpandBlob(pIn1); applyAffinity(pIn1, cAff, encoding); pIn1++; } @@ -69036,8 +70358,9 @@ case OP_MakeRecord: { if( zAffinity ){ pRec = pData0; do{ - applyAffinity(pRec, *(zAffinity++), encoding); - }while( (++pRec)<=pLast ); + applyAffinity(pRec++, *(zAffinity++), encoding); + assert( zAffinity[0]==0 || pRec<=pLast ); + }while( zAffinity[0] ); } /* Loop through the elements that will make up the record to figure @@ -69104,7 +70427,7 @@ case OP_MakeRecord: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; - pOut->flags = MEM_Blob | MEM_Dyn; + pOut->flags = MEM_Blob; pOut->xDel = 0; if( nZero ){ pOut->u.nZero = nZero; @@ -69381,25 +70704,19 @@ case OP_AutoCommit: { break; } -/* Opcode: Transaction P1 P2 * * * +/* Opcode: Transaction P1 P2 P3 P4 P5 ** -** Begin a transaction. The transaction ends when a Commit or Rollback -** opcode is encountered. Depending on the ON CONFLICT setting, the -** transaction might also be rolled back if an error is encountered. +** Begin a transaction on database P1 if a transaction is not already +** active. +** If P2 is non-zero, then a write-transaction is started, or if a +** read-transaction is already active, it is upgraded to a write-transaction. +** If P2 is zero, then a read-transaction is started. ** ** P1 is the index of the database file on which the transaction is ** started. Index 0 is the main database file and index 1 is the ** file used for temporary tables. Indices of 2 or more are used for ** attached databases. ** -** If P2 is non-zero, then a write-transaction is started. A RESERVED lock is -** obtained on the database file when a write-transaction is started. No -** other process can start another write transaction while this transaction is -** underway. Starting a write transaction also creates a rollback journal. A -** write transaction must be started before any changes can be made to the -** database. If P2 is greater than or equal to 2 then an EXCLUSIVE lock is -** also obtained on the file. -** ** If a write-transaction is started and the Vdbe.usesStmtJournal flag is ** true (this flag is set if the Vdbe may modify more than one row and may ** throw an ABORT exception), a statement transaction may also be opened. @@ -69410,15 +70727,26 @@ case OP_AutoCommit: { ** entire transaction. If no error is encountered, the statement transaction ** will automatically commit when the VDBE halts. ** -** If P2 is zero, then a read-lock is obtained on the database file. +** If P5!=0 then this opcode also checks the schema cookie against P3 +** and the schema generation counter against P4. +** The cookie changes its value whenever the database schema changes. +** This operation is used to detect when that the cookie has changed +** and that the current process needs to reread the schema. If the schema +** cookie in P3 differs from the schema cookie in the database header or +** if the schema generation counter in P4 differs from the current +** generation counter, then an SQLITE_SCHEMA error is raised and execution +** halts. The sqlite3_step() wrapper function might then reprepare the +** statement and rerun it from the beginning. */ case OP_Transaction: { Btree *pBt; + int iMeta; + int iGen; assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ rc = SQLITE_READONLY; goto abort_due_to_error; @@ -69457,115 +70785,15 @@ case OP_Transaction: { p->nStmtDefCons = db->nDeferredCons; p->nStmtDefImmCons = db->nDeferredImmCons; } - } - break; -} - -/* Opcode: ReadCookie P1 P2 P3 * * -** -** Read cookie number P3 from database P1 and write it into register P2. -** P3==1 is the schema version. P3==2 is the database format. -** P3==3 is the recommended pager cache size, and so forth. P1==0 is -** the main database file and P1==1 is the database file used to store -** temporary tables. -** -** There must be a read-lock on the database (either a transaction -** must be started or there must be an open cursor) before -** executing this instruction. -*/ -case OP_ReadCookie: { /* out2-prerelease */ - int iMeta; - int iDb; - int iCookie; - - assert( p->bIsReader ); - iDb = pOp->p1; - iCookie = pOp->p3; - assert( pOp->p3=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 ); - assert( (p->btreeMask & (((yDbMask)1)<aDb[iDb].pBt, iCookie, (u32 *)&iMeta); - pOut->u.i = iMeta; - break; -} -/* Opcode: SetCookie P1 P2 P3 * * -** -** Write the content of register P3 (interpreted as an integer) -** into cookie number P2 of database P1. P2==1 is the schema version. -** P2==2 is the database format. P2==3 is the recommended pager cache -** size, and so forth. P1==0 is the main database file and P1==1 is the -** database file used to store temporary tables. -** -** A transaction must be started before executing this opcode. -*/ -case OP_SetCookie: { /* in3 */ - Db *pDb; - assert( pOp->p2p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); - assert( p->readOnly==0 ); - pDb = &db->aDb[pOp->p1]; - assert( pDb->pBt!=0 ); - assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); - pIn3 = &aMem[pOp->p3]; - sqlite3VdbeMemIntegerify(pIn3); - /* See note about index shifting on OP_ReadCookie */ - rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); - if( pOp->p2==BTREE_SCHEMA_VERSION ){ - /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = (int)pIn3->u.i; - db->flags |= SQLITE_InternChanges; - }else if( pOp->p2==BTREE_FILE_FORMAT ){ - /* Record changes in the file format */ - pDb->pSchema->file_format = (u8)pIn3->u.i; - } - if( pOp->p1==1 ){ - /* Invalidate all prepared statements whenever the TEMP database - ** schema is changed. Ticket #1644 */ - sqlite3ExpirePreparedStatements(db); - p->expired = 0; - } - break; -} - -/* Opcode: VerifyCookie P1 P2 P3 * * -** -** Check the value of global database parameter number 0 (the -** schema version) and make sure it is equal to P2 and that the -** generation counter on the local schema parse equals P3. -** -** P1 is the database number which is 0 for the main database file -** and 1 for the file holding temporary tables and some higher number -** for auxiliary databases. -** -** The cookie changes its value whenever the database schema changes. -** This operation is used to detect when that the cookie has changed -** and that the current process needs to reread the schema. -** -** Either a transaction needs to have been started or an OP_Open needs -** to be executed (to establish a read lock) before this opcode is -** invoked. -*/ -case OP_VerifyCookie: { - int iMeta; - int iGen; - Btree *pBt; - - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); - assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); - assert( p->bIsReader ); - pBt = db->aDb[pOp->p1].pBt; - if( pBt ){ + /* Gather the schema version number for checking */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); iGen = db->aDb[pOp->p1].pSchema->iGeneration; }else{ iGen = iMeta = 0; } - if( iMeta!=pOp->p2 || iGen!=pOp->p3 ){ + assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); + if( pOp->p5 && (iMeta!=pOp->p3 || iGen!=pOp->p4.i) ){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); /* If the schema-cookie from the database file matches the cookie @@ -69584,13 +70812,82 @@ case OP_VerifyCookie: { if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){ sqlite3ResetOneSchema(db, pOp->p1); } - p->expired = 1; rc = SQLITE_SCHEMA; } break; } +/* Opcode: ReadCookie P1 P2 P3 * * +** +** Read cookie number P3 from database P1 and write it into register P2. +** P3==1 is the schema version. P3==2 is the database format. +** P3==3 is the recommended pager cache size, and so forth. P1==0 is +** the main database file and P1==1 is the database file used to store +** temporary tables. +** +** There must be a read-lock on the database (either a transaction +** must be started or there must be an open cursor) before +** executing this instruction. +*/ +case OP_ReadCookie: { /* out2-prerelease */ + int iMeta; + int iDb; + int iCookie; + + assert( p->bIsReader ); + iDb = pOp->p1; + iCookie = pOp->p3; + assert( pOp->p3=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + assert( DbMaskTest(p->btreeMask, iDb) ); + + sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); + pOut->u.i = iMeta; + break; +} + +/* Opcode: SetCookie P1 P2 P3 * * +** +** Write the content of register P3 (interpreted as an integer) +** into cookie number P2 of database P1. P2==1 is the schema version. +** P2==2 is the database format. P2==3 is the recommended pager cache +** size, and so forth. P1==0 is the main database file and P1==1 is the +** database file used to store temporary tables. +** +** A transaction must be started before executing this opcode. +*/ +case OP_SetCookie: { /* in3 */ + Db *pDb; + assert( pOp->p2p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pDb = &db->aDb[pOp->p1]; + assert( pDb->pBt!=0 ); + assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + pIn3 = &aMem[pOp->p3]; + sqlite3VdbeMemIntegerify(pIn3); + /* See note about index shifting on OP_ReadCookie */ + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, (int)pIn3->u.i); + if( pOp->p2==BTREE_SCHEMA_VERSION ){ + /* When the schema cookie changes, record the new cookie internally */ + pDb->pSchema->schema_cookie = (int)pIn3->u.i; + db->flags |= SQLITE_InternChanges; + }else if( pOp->p2==BTREE_FILE_FORMAT ){ + /* Record changes in the file format */ + pDb->pSchema->file_format = (u8)pIn3->u.i; + } + if( pOp->p1==1 ){ + /* Invalidate all prepared statements whenever the TEMP database + ** schema is changed. Ticket #1644 */ + sqlite3ExpirePreparedStatements(db); + p->expired = 0; + } + break; +} + /* Opcode: OpenRead P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** @@ -69620,7 +70917,21 @@ case OP_VerifyCookie: { ** sequence of the index being opened. Otherwise, if P4 is an integer ** value, it is set to the number of columns in the table. ** -** See also OpenWrite. +** See also: OpenWrite, ReopenIdx +*/ +/* Opcode: ReopenIdx P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** The ReopenIdx opcode works exactly like ReadOpen except that it first +** checks to see if the cursor on P1 is already open with a root page +** number of P2 and if it is this opcode becomes a no-op. In other words, +** if the cursor is already open, do not reopen it. +** +** The ReopenIdx opcode may only be used with P5==0 and with P4 being +** a P4_KEYINFO object. Furthermore, the P3 value must be the same as +** every other ReopenIdx or OpenRead for the same cursor number. +** +** See the OpenRead opcode documentation for additional information. */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 @@ -69642,6 +70953,19 @@ case OP_VerifyCookie: { ** ** See also OpenRead. */ +case OP_ReopenIdx: { + VdbeCursor *pCur; + + assert( pOp->p5==0 ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + break; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +} case OP_OpenRead: case OP_OpenWrite: { int nField; @@ -69656,7 +70980,8 @@ case OP_OpenWrite: { assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); assert( p->bIsReader ); - assert( pOp->opcode==OP_OpenRead || p->readOnly==0 ); + assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx + || p->readOnly==0 ); if( p->expired ){ rc = SQLITE_ABORT; @@ -69668,7 +70993,7 @@ case OP_OpenWrite: { p2 = pOp->p2; iDb = pOp->p3; assert( iDb>=0 && iDbnDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); pDb = &db->aDb[iDb]; pX = pDb->pBt; assert( pX!=0 ); @@ -69713,6 +71038,7 @@ case OP_OpenWrite: { if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; + pCur->pgnoRoot = p2; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); @@ -69772,6 +71098,7 @@ case OP_OpenEphemeral: { pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; + pCx->isEphemeral = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ @@ -69804,7 +71131,7 @@ case OP_OpenEphemeral: { break; } -/* Opcode: SorterOpen P1 * * P4 * +/* Opcode: SorterOpen P1 P2 * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large @@ -69824,14 +71151,13 @@ case OP_SorterOpen: { break; } -/* Opcode: OpenPseudo P1 P2 P3 * P5 -** Synopsis: content in r[P2@P3] +/* Opcode: OpenPseudo P1 P2 P3 * * +** Synopsis: P3 columns in r[P2] ** ** Open a new cursor that points to a fake table that contains a single -** row of data. The content of that one row in the content of memory -** register P2 when P5==0. In other words, cursor P1 becomes an alias for the -** MEM_Blob content contained in register P2. When P5==1, then the -** row is represented by P3 consecutive registers beginning with P2. +** row of data. The content of that one row is the content of memory +** register P2. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. ** ** A pseudo-table created by this opcode is used to hold a single ** row output from the sorter so that the row can be decomposed into @@ -69851,7 +71177,7 @@ case OP_OpenPseudo: { pCx->nullRow = 1; pCx->pseudoTableReg = pOp->p2; pCx->isTable = 1; - pCx->multiPseudo = pOp->p5; + assert( pOp->p5==0 ); break; } @@ -69879,7 +71205,7 @@ case OP_Close: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** -** See also: Found, NotFound, Distinct, SeekLt, SeekGt, SeekLe +** See also: Found, NotFound, SeekLt, SeekGt, SeekLe */ /* Opcode: SeekGt P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -69893,7 +71219,7 @@ case OP_Close: { ** is greater than the key value. If there are no records greater than ** the key and P2 is not zero, then jump to P2. ** -** See also: Found, NotFound, Distinct, SeekLt, SeekGe, SeekLe +** See also: Found, NotFound, SeekLt, SeekGe, SeekLe */ /* Opcode: SeekLt P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -69907,7 +71233,7 @@ case OP_Close: { ** is less than the key value. If there are no records less than ** the key and P2 is not zero, then jump to P2. ** -** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLe +** See also: Found, NotFound, SeekGt, SeekGe, SeekLe */ /* Opcode: SeekLe P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -69921,12 +71247,12 @@ case OP_Close: { ** is less than or equal to the key value. If there are no records ** less than or equal to the key and P2 is not zero, then jump to P2. ** -** See also: Found, NotFound, Distinct, SeekGt, SeekGe, SeekLt +** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLt: /* jump, in3 */ -case OP_SeekLe: /* jump, in3 */ -case OP_SeekGe: /* jump, in3 */ -case OP_SeekGt: { /* jump, in3 */ +case OP_SeekLT: /* jump, in3 */ +case OP_SeekLE: /* jump, in3 */ +case OP_SeekGE: /* jump, in3 */ +case OP_SeekGT: { /* jump, in3 */ int res; int oc; VdbeCursor *pC; @@ -69939,9 +71265,9 @@ case OP_SeekGt: { /* jump, in3 */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->pseudoTableReg==0 ); - assert( OP_SeekLe == OP_SeekLt+1 ); - assert( OP_SeekGe == OP_SeekLt+2 ); - assert( OP_SeekGt == OP_SeekLt+3 ); + assert( OP_SeekLE == OP_SeekLT+1 ); + assert( OP_SeekGE == OP_SeekLT+2 ); + assert( OP_SeekGT == OP_SeekLT+3 ); assert( pC->isOrdered ); assert( pC->pCursor!=0 ); oc = pOp->opcode; @@ -69961,7 +71287,7 @@ case OP_SeekGt: { /* jump, in3 */ if( (pIn3->flags & MEM_Real)==0 ){ /* If the P3 value cannot be converted into any kind of a number, ** then the seek is not possible, so jump to P2 */ - pc = pOp->p2 - 1; + pc = pOp->p2 - 1; VdbeBranchTaken(1,2); break; } @@ -69973,19 +71299,19 @@ case OP_SeekGt: { /* jump, in3 */ ** (x <= 4.9) -> (x < 5) */ if( pIn3->r<(double)iKey ){ - assert( OP_SeekGe==(OP_SeekGt-1) ); - assert( OP_SeekLt==(OP_SeekLe-1) ); - assert( (OP_SeekLe & 0x0001)==(OP_SeekGt & 0x0001) ); - if( (oc & 0x0001)==(OP_SeekGt & 0x0001) ) oc--; + assert( OP_SeekGE==(OP_SeekGT-1) ); + assert( OP_SeekLT==(OP_SeekLE-1) ); + assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekGT & 0x0001) ) oc--; } /* If the approximation iKey is smaller than the actual real search ** term, substitute <= for < and > for >=. */ else if( pIn3->r>(double)iKey ){ - assert( OP_SeekLe==(OP_SeekLt+1) ); - assert( OP_SeekGt==(OP_SeekGe+1) ); - assert( (OP_SeekLt & 0x0001)==(OP_SeekGe & 0x0001) ); - if( (oc & 0x0001)==(OP_SeekLt & 0x0001) ) oc++; + assert( OP_SeekLE==(OP_SeekLT+1) ); + assert( OP_SeekGT==(OP_SeekGE+1) ); + assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); @@ -70004,17 +71330,17 @@ case OP_SeekGt: { /* jump, in3 */ r.nField = (u16)nField; /* The next line of code computes as follows, only faster: - ** if( oc==OP_SeekGt || oc==OP_SeekLe ){ - ** r.flags = UNPACKED_INCRKEY; + ** if( oc==OP_SeekGT || oc==OP_SeekLE ){ + ** r.default_rc = -1; ** }else{ - ** r.flags = 0; + ** r.default_rc = +1; ** } */ - r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); - assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); - assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); - assert( oc!=OP_SeekGe || r.flags==0 ); - assert( oc!=OP_SeekLt || r.flags==0 ); + r.default_rc = ((1 & (oc - OP_SeekLT)) ? -1 : +1); + assert( oc!=OP_SeekGT || r.default_rc==-1 ); + assert( oc!=OP_SeekLE || r.default_rc==-1 ); + assert( oc!=OP_SeekGE || r.default_rc==+1 ); + assert( oc!=OP_SeekLT || r.default_rc==+1 ); r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG @@ -70032,8 +71358,9 @@ case OP_SeekGt: { /* jump, in3 */ #ifdef SQLITE_TEST sqlite3_search_count++; #endif - if( oc>=OP_SeekGe ){ assert( oc==OP_SeekGe || oc==OP_SeekGt ); - if( res<0 || (res==0 && oc==OP_SeekGt) ){ + if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); + if( res<0 || (res==0 && oc==OP_SeekGT) ){ + res = 0; rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; pC->rowidIsValid = 0; @@ -70041,8 +71368,9 @@ case OP_SeekGt: { /* jump, in3 */ res = 0; } }else{ - assert( oc==OP_SeekLt || oc==OP_SeekLe ); - if( res>0 || (res==0 && oc==OP_SeekLt) ){ + assert( oc==OP_SeekLT || oc==OP_SeekLE ); + if( res>0 || (res==0 && oc==OP_SeekLT) ){ + res = 0; rc = sqlite3BtreePrevious(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; pC->rowidIsValid = 0; @@ -70054,6 +71382,7 @@ case OP_SeekGt: { /* jump, in3 */ } } assert( pOp->p2>0 ); + VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; } @@ -70162,16 +71491,13 @@ case OP_Found: { /* jump, in3 */ r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; r.aMem = pIn3; + for(ii=0; iip3+i, &r.aMem[i]); - } - } + if( ii ) REGISTER_TRACE(pOp->p3+ii, &r.aMem[ii]); #endif - r.flags = UNPACKED_PREFIX_MATCH; + } pIdxKey = &r; }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( @@ -70181,15 +71507,15 @@ case OP_Found: { /* jump, in3 */ assert( pIn3->flags & MEM_Blob ); assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); - pIdxKey->flags |= UNPACKED_PREFIX_MATCH; } + pIdxKey->default_rc = 0; if( pOp->opcode==OP_NoConflict ){ /* For the OP_NoConflict opcode, take the jump if any of the ** input fields are NULL, since any key with a NULL will not ** conflict */ for(ii=0; iip2 - 1; + pc = pOp->p2 - 1; VdbeBranchTaken(1,2); break; } } @@ -70207,8 +71533,10 @@ case OP_Found: { /* jump, in3 */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; if( pOp->opcode==OP_Found ){ + VdbeBranchTaken(alreadyExists!=0,2); if( alreadyExists ) pc = pOp->p2 - 1; }else{ + VdbeBranchTaken(alreadyExists==0,2); if( !alreadyExists ) pc = pOp->p2 - 1; } break; @@ -70251,6 +71579,7 @@ case OP_NotExists: { /* jump, in3 */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; + VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; assert( pC->rowidIsValid==0 ); @@ -70260,7 +71589,7 @@ case OP_NotExists: { /* jump, in3 */ } /* Opcode: Sequence P1 P2 * * * -** Synopsis: r[P2]=rowid +** Synopsis: r[P2]=cursor[P1].ctr++ ** ** Find the next available sequence number for cursor P1. ** Write the sequence number into register P2. @@ -70332,59 +71661,54 @@ case OP_NewRowid: { /* out2-prerelease */ #endif if( !pC->useRandomRowid ){ - v = sqlite3BtreeGetCachedRowid(pC->pCursor); - if( v==0 ){ - rc = sqlite3BtreeLast(pC->pCursor, &res); - if( rc!=SQLITE_OK ){ - goto abort_due_to_error; - } - if( res ){ - v = 1; /* IMP: R-61914-48074 */ + rc = sqlite3BtreeLast(pC->pCursor, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( res ){ + v = 1; /* IMP: R-61914-48074 */ + }else{ + assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ + if( v>=MAX_ROWID ){ + pC->useRandomRowid = 1; }else{ - assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); - rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ - if( v>=MAX_ROWID ){ - pC->useRandomRowid = 1; - }else{ - v++; /* IMP: R-29538-34987 */ - } + v++; /* IMP: R-29538-34987 */ } } + } #ifndef SQLITE_OMIT_AUTOINCREMENT - if( pOp->p3 ){ + if( pOp->p3 ){ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3>0 ); + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3>0 ); - if( p->pFrame ){ - for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); - /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=pFrame->nMem ); - pMem = &pFrame->aMem[pOp->p3]; - }else{ - /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=(p->nMem-p->nCursor) ); - pMem = &aMem[pOp->p3]; - memAboutToChange(p, pMem); - } - assert( memIsValid(pMem) ); - - REGISTER_TRACE(pOp->p3, pMem); - sqlite3VdbeMemIntegerify(pMem); - assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ - if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ - rc = SQLITE_FULL; /* IMP: R-12275-61338 */ - goto abort_due_to_error; - } - if( vu.i+1 ){ - v = pMem->u.i + 1; - } - pMem->u.i = v; + assert( pOp->p3<=pFrame->nMem ); + pMem = &pFrame->aMem[pOp->p3]; + }else{ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=(p->nMem-p->nCursor) ); + pMem = &aMem[pOp->p3]; + memAboutToChange(p, pMem); } -#endif + assert( memIsValid(pMem) ); - sqlite3BtreeSetCachedRowid(pC->pCursor, vp3, pMem); + sqlite3VdbeMemIntegerify(pMem); + assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ + if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ + rc = SQLITE_FULL; /* IMP: R-12275-61338 */ + goto abort_due_to_error; + } + if( vu.i+1 ){ + v = pMem->u.i + 1; + } + pMem->u.i = v; } +#endif if( pC->useRandomRowid ){ /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the ** largest possible integer (9223372036854775807) then the database @@ -70518,7 +71842,6 @@ case OP_InsertInt: { }else{ nZero = 0; } - sqlite3BtreeSetCachedRowid(pC->pCursor, 0); rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey, pData->z, pData->n, nZero, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult @@ -70580,7 +71903,6 @@ case OP_Delete: { rc = sqlite3VdbeCursorMoveto(pC); if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; - sqlite3BtreeSetCachedRowid(pC->pCursor, 0); rc = sqlite3BtreeDelete(pC->pCursor); pC->cacheStatus = CACHE_STALE; @@ -70632,6 +71954,7 @@ case OP_SorterCompare: { pIn3 = &aMem[pOp->p3]; nIgnore = pOp->p4.i; rc = sqlite3VdbeSorterCompare(pC, pIn3, nIgnore, &res); + VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2-1; } @@ -70650,6 +71973,7 @@ case OP_SorterData: { pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); break; } @@ -70669,7 +71993,7 @@ case OP_SorterData: { ** ** Write into register P2 the complete row key for cursor P1. ** There is no interpretation of the data. -** The key is copied onto the P3 register exactly as +** The key is copied onto the P2 register exactly as ** it is found in the database file. ** ** If the P1 cursor must be pointing to a valid row (not a NULL row) @@ -70831,8 +72155,9 @@ case OP_Last: { /* jump */ pC->deferredMoveto = 0; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - if( pOp->p2>0 && res ){ - pc = pOp->p2 - 1; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) pc = pOp->p2 - 1; } break; } @@ -70889,13 +72214,14 @@ case OP_Rewind: { /* jump */ } pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); + VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; } break; } -/* Opcode: Next P1 P2 * * P5 +/* Opcode: Next P1 P2 P3 P4 P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through @@ -70905,6 +72231,11 @@ case OP_Rewind: { /* jump */ ** The P1 cursor must be for a real table, not a pseudo-table. P1 must have ** been opened prior to this opcode or the program will segfault. ** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** ** P4 is always of type P4_ADVANCE. The function pointer points to ** sqlite3BtreeNext(). ** @@ -70913,12 +72244,12 @@ case OP_Rewind: { /* jump */ ** ** See also: Prev, NextIfOpen */ -/* Opcode: NextIfOpen P1 P2 * * P5 +/* Opcode: NextIfOpen P1 P2 P3 P4 P5 ** ** This opcode works just like OP_Next except that if cursor P1 is not ** open it behaves a no-op. */ -/* Opcode: Prev P1 P2 * * P5 +/* Opcode: Prev P1 P2 P3 P4 P5 ** ** Back up cursor P1 so that it points to the previous key/data pair in its ** table or index. If there is no previous key/value pairs then fall through @@ -70928,13 +72259,18 @@ case OP_Rewind: { /* jump */ ** The P1 cursor must be for a real table, not a pseudo-table. If P1 is ** not open then the behavior is undefined. ** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** ** P4 is always of type P4_ADVANCE. The function pointer points to ** sqlite3BtreePrevious(). ** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ -/* Opcode: PrevIfOpen P1 P2 * * P5 +/* Opcode: PrevIfOpen P1 P2 P3 P4 P5 ** ** This opcode works just like OP_Prev except that if cursor P1 is not ** open it behaves a no-op. @@ -70945,6 +72281,7 @@ case OP_SorterNext: { /* jump */ pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); + res = 0; rc = sqlite3VdbeSorterNext(db, pC, &res); goto next_tail; case OP_PrevIfOpen: /* jump */ @@ -70956,9 +72293,12 @@ case OP_Next: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5aCounter) ); pC = p->apCsr[pOp->p1]; + res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); assert( pC->pCursor ); + assert( res==0 || (res==1 && pC->isTable==0) ); + testcase( res==1 ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); @@ -70966,6 +72306,7 @@ case OP_Next: /* jump */ rc = pOp->p4.xAdvance(pC->pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; + VdbeBranchTaken(res==0,2); if( res==0 ){ pC->nullRow = 0; pc = pOp->p2 - 1; @@ -70990,6 +72331,14 @@ case OP_Next: /* jump */ ** P3 is a flag that provides a hint to the b-tree layer that this ** insert is likely to be an append. ** +** If P5 has the OPFLAG_NCHANGE bit set, then the change counter is +** incremented by this instruction. If the OPFLAG_NCHANGE bit is clear, +** then the change counter is unchanged. +** +** If P5 has the OPFLAG_USESEEKRESULT bit set, then the cursor must have +** just done a seek to the spot where the new entry is to be inserted. +** This flag avoids doing an extra seek. +** ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ @@ -71050,7 +72399,7 @@ case OP_IdxDelete: { assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; - r.flags = UNPACKED_PREFIX_MATCH; + r.default_rc = 0; r.aMem = &aMem[pOp->p2]; #ifdef SQLITE_DEBUG { int i; for(i=0; ip4type==P4_INT32 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; - if( pOp->p5 ){ - r.flags = UNPACKED_INCRKEY | UNPACKED_PREFIX_MATCH; + if( pOp->opcodeopcode==OP_IdxLE || pOp->opcode==OP_IdxGT ); + r.default_rc = -1; }else{ - r.flags = UNPACKED_PREFIX_MATCH; + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxLT ); + r.default_rc = 0; } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG @@ -71155,12 +72524,15 @@ case OP_IdxGE: { /* jump */ #endif res = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3VdbeIdxKeyCompare(pC, &r, &res); - if( pOp->opcode==OP_IdxLT ){ + assert( (OP_IdxLE&1)==(OP_IdxLT&1) && (OP_IdxGE&1)==(OP_IdxGT&1) ); + if( (pOp->opcode&1)==(OP_IdxLT&1) ){ + assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); res = -res; }else{ - assert( pOp->opcode==OP_IdxGE ); + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxGT ); res++; } + VdbeBranchTaken(res>0,2); if( res>0 ){ pc = pOp->p2 - 1 ; } @@ -71213,7 +72585,7 @@ case OP_Destroy: { /* out2-prerelease */ }else{ iDb = pOp->p3; assert( iCnt==1 ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); iMoved = 0; /* Not needed. Only to silence a warning. */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; @@ -71253,8 +72625,7 @@ case OP_Clear: { nChange = 0; assert( p->readOnly==0 ); - assert( pOp->p1!=1 ); - assert( (p->btreeMask & (((yDbMask)1)<p2))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); rc = sqlite3BtreeClearTable( db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) ); @@ -71269,6 +72640,29 @@ case OP_Clear: { break; } +/* Opcode: ResetSorter P1 * * * * +** +** Delete all contents from the ephemeral table or sorter +** that is open on cursor P1. +** +** This opcode only works for cursors used for sorting and +** opened with OP_OpenEphemeral or OP_SorterOpen. +*/ +case OP_ResetSorter: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( pC->pSorter ){ + sqlite3VdbeSorterReset(db, pC->pSorter); + }else{ + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + } + break; +} + /* Opcode: CreateTable P1 P2 * * * ** Synopsis: r[P2]=root iDb=P1 ** @@ -71301,7 +72695,7 @@ case OP_CreateTable: { /* out2-prerelease */ pgno = 0; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); @@ -71466,7 +72860,7 @@ case OP_IntegrityCk: { } aRoot[j] = 0; assert( pOp->p5nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p5))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); sqlite3DbFree(db, aRoot); @@ -71522,9 +72916,11 @@ case OP_RowSetRead: { /* jump, in1, out3 */ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); pc = pOp->p2 - 1; + VdbeBranchTaken(1,2); }else{ /* A value was pulled from the index */ sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); + VdbeBranchTaken(0,2); } goto check_for_interrupt; } @@ -71573,9 +72969,8 @@ case OP_RowSetTest: { /* jump, in1, in3 */ assert( pOp->p4type==P4_INT32 ); assert( iSet==-1 || iSet>=0 ); if( iSet ){ - exists = sqlite3RowSetTest(pIn1->u.pRowSet, - (u8)(iSet>=0 ? iSet & 0xf : 0xff), - pIn3->u.i); + exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); + VdbeBranchTaken(exists!=0,2); if( exists ){ pc = pOp->p2 - 1; break; @@ -71590,7 +72985,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ #ifndef SQLITE_OMIT_TRIGGER -/* Opcode: Program P1 P2 P3 P4 * +/* Opcode: Program P1 P2 P3 P4 P5 ** ** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). ** @@ -71602,6 +72997,8 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** memory required by the sub-vdbe at runtime. ** ** P4 is a pointer to the VM containing the trigger program. +** +** If P5 is non-zero, then recursive program invocation is enabled. */ case OP_Program: { /* jump */ int nMem; /* Number of memory registers for sub-program */ @@ -71679,7 +73076,7 @@ case OP_Program: { /* jump */ pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ - pMem->flags = MEM_Invalid; + pMem->flags = MEM_Undefined; pMem->db = db; } }else{ @@ -71766,8 +73163,10 @@ case OP_FkCounter: { */ case OP_FkIfZero: { /* jump */ if( pOp->p1 ){ + VdbeBranchTaken(db->nDeferredCons==0 && db->nDeferredImmCons==0, 2); if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; }else{ + VdbeBranchTaken(p->nFkConstraint==0 && db->nDeferredImmCons==0, 2); if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) pc = pOp->p2-1; } break; @@ -71816,6 +73215,7 @@ case OP_MemMax: { /* in2 */ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken( pIn1->u.i>0, 2); if( pIn1->u.i>0 ){ pc = pOp->p2 - 1; } @@ -71833,6 +73233,7 @@ case OP_IfPos: { /* jump, in1 */ case OP_IfNeg: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i<0 ){ pc = pOp->p2 - 1; } @@ -71852,6 +73253,7 @@ case OP_IfZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); pIn1->u.i += pOp->p3; + VdbeBranchTaken(pIn1->u.i==0, 2); if( pIn1->u.i==0 ){ pc = pOp->p2 - 1; } @@ -71886,7 +73288,6 @@ case OP_AggStep: { assert( memIsValid(pRec) ); apVal[i] = pRec; memAboutToChange(p, pRec); - sqlite3VdbeMemStoreType(pRec); } ctx.pFunc = pOp->p4.pFunc; assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); @@ -71989,7 +73390,7 @@ case OP_Checkpoint: { #endif #ifndef SQLITE_OMIT_PRAGMA -/* Opcode: JournalMode P1 P2 P3 * P5 +/* Opcode: JournalMode P1 P2 P3 * * ** ** Change the journal mode of database P1 to P3. P3 must be one of the ** PAGER_JOURNALMODE_XXX values. If changing between the various rollback @@ -72119,10 +73520,11 @@ case OP_IncrVacuum: { /* jump */ Btree *pBt; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); + VdbeBranchTaken(rc==SQLITE_DONE,2); if( rc==SQLITE_DONE ){ pc = pOp->p2 - 1; rc = SQLITE_OK; @@ -72133,12 +73535,13 @@ case OP_IncrVacuum: { /* jump */ /* Opcode: Expire P1 * * * * ** -** Cause precompiled statements to become expired. An expired statement -** fails with an error code of SQLITE_SCHEMA if it is ever executed -** (via sqlite3_step()). +** Cause precompiled statements to expire. When an expired statement +** is executed using sqlite3_step() it will either automatically +** reprepare itself (if it was originally created using sqlite3_prepare_v2()) +** or it will fail with SQLITE_SCHEMA. ** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, -** then only the currently executing statement is affected. +** then only the currently executing statement is expired. */ case OP_Expire: { if( !pOp->p1 ){ @@ -72170,7 +73573,7 @@ case OP_TableLock: { if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ int p1 = pOp->p1; assert( p1>=0 && p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); if( (rc&0xFF)==SQLITE_LOCKED ){ @@ -72267,7 +73670,7 @@ case OP_VOpen: { #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VFilter P1 P2 P3 P4 * -** Synopsis: iPlan=r[P3] zPlan='P4' +** Synopsis: iplan=r[P3] zplan='P4' ** ** P1 is a cursor opened using VOpen. P2 is an address to jump to if ** the filtered result set is empty. @@ -72319,7 +73722,6 @@ case OP_VFilter: { /* jump */ apArg = p->apArg; for(i = 0; iinVtabMethod = 1; @@ -72329,7 +73731,7 @@ case OP_VFilter: { /* jump */ if( rc==SQLITE_OK ){ res = pModule->xEof(pVtabCursor); } - + VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; } @@ -72434,7 +73836,7 @@ case OP_VNext: { /* jump */ if( rc==SQLITE_OK ){ res = pModule->xEof(pCur->pVtabCursor); } - + VdbeBranchTaken(!res,2); if( !res ){ /* If there is data, jump to P2 */ pc = pOp->p2 - 1; @@ -72475,7 +73877,7 @@ case OP_VRename: { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE -/* Opcode: VUpdate P1 P2 P3 P4 * +/* Opcode: VUpdate P1 P2 P3 P4 P5 ** Synopsis: data=r[P3@P2] ** ** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. @@ -72498,6 +73900,9 @@ case OP_VRename: { ** P1 is a boolean flag. If it is set to true and the xUpdate call ** is successful, then the value returned by sqlite3_last_insert_rowid() ** is set to the value of the rowid for the row just inserted. +** +** P5 is the error actions (OE_Replace, OE_Fail, OE_Ignore, etc) to +** apply in the case of a constraint failure on an insert or update. */ case OP_VUpdate: { sqlite3_vtab *pVtab; @@ -72523,7 +73928,6 @@ case OP_VUpdate: { for(i=0; ip2 ){ + pc = pOp->p2 - 1; + } +#ifndef SQLITE_OMIT_TRACE if( db->xTrace && !p->doingRerun && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 @@ -72609,7 +74023,7 @@ case OP_Trace: { if( zTrace ){ int i; for(i=0; inDb; i++){ - if( MASKBIT(i) & p->btreeMask)==0 ) continue; + if( DbMaskTest(p->btreeMask, i)==0 ) continue; sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); } } @@ -72621,9 +74035,9 @@ case OP_Trace: { sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); } #endif /* SQLITE_DEBUG */ +#endif /* SQLITE_OMIT_TRACE */ break; } -#endif /* Opcode: Noop * * * * * @@ -72652,13 +74066,9 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { - u64 elapsed = sqlite3Hwtime() - start; - pOp->cycles += elapsed; + u64 endTime = sqlite3Hwtime(); + if( endTime>start ) pOp->cycles += endTime - start; pOp->cnt++; -#if 0 - fprintf(stdout, "%10llu ", elapsed); - sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]); -#endif } #endif @@ -72828,9 +74238,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ p->iOffset = pC->aType[p->iCol + pC->nField]; p->nByte = sqlite3VdbeSerialTypeLen(type); p->pCsr = pC->pCursor; - sqlite3BtreeEnterCursor(p->pCsr); - sqlite3BtreeCacheOverflow(p->pCsr); - sqlite3BtreeLeaveCursor(p->pCsr); + sqlite3BtreeIncrblobCursor(p->pCsr); } } @@ -72884,22 +74292,20 @@ SQLITE_API int sqlite3_blob_open( ** which closes the b-tree cursor and (possibly) commits the ** transaction. */ + static const int iLn = VDBE_OFFSET_LINENO(4); static const VdbeOpList openBlob[] = { - {OP_Transaction, 0, 0, 0}, /* 0: Start a transaction */ - {OP_VerifyCookie, 0, 0, 0}, /* 1: Check the schema cookie */ - {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ - + /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ + {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ /* One of the following two instructions is replaced by an OP_Noop. */ - {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ - {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ - - {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ - {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 7 */ - {OP_ResultRow, 1, 0, 0}, /* 8 */ - {OP_Goto, 0, 5, 0}, /* 9 */ - {OP_Close, 0, 0, 0}, /* 10 */ - {OP_Halt, 0, 0, 0}, /* 11 */ + {OP_OpenRead, 0, 0, 0}, /* 2: Open cursor 0 for reading */ + {OP_OpenWrite, 0, 0, 0}, /* 3: Open cursor 0 for read/write */ + {OP_Variable, 1, 1, 1}, /* 4: Push the rowid to the stack */ + {OP_NotExists, 0, 10, 1}, /* 5: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 6 */ + {OP_ResultRow, 1, 0, 0}, /* 7 */ + {OP_Goto, 0, 4, 0}, /* 8 */ + {OP_Close, 0, 0, 0}, /* 9 */ + {OP_Halt, 0, 0, 0}, /* 10 */ }; int rc = SQLITE_OK; @@ -73012,36 +74418,31 @@ SQLITE_API int sqlite3_blob_open( Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); - - /* Configure the OP_Transaction */ - sqlite3VdbeChangeP1(v, 0, iDb); - sqlite3VdbeChangeP2(v, 0, flags); - - /* Configure the OP_VerifyCookie */ - sqlite3VdbeChangeP1(v, 1, iDb); - sqlite3VdbeChangeP2(v, 1, pTab->pSchema->schema_cookie); - sqlite3VdbeChangeP3(v, 1, pTab->pSchema->iGeneration); + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, + pTab->pSchema->schema_cookie, + pTab->pSchema->iGeneration); + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ sqlite3VdbeUsesBtree(v, iDb); /* Configure the OP_TableLock instruction */ #ifdef SQLITE_OMIT_SHARED_CACHE - sqlite3VdbeChangeToNoop(v, 2); + sqlite3VdbeChangeToNoop(v, 1); #else - sqlite3VdbeChangeP1(v, 2, iDb); - sqlite3VdbeChangeP2(v, 2, pTab->tnum); - sqlite3VdbeChangeP3(v, 2, flags); - sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT); + sqlite3VdbeChangeP1(v, 1, iDb); + sqlite3VdbeChangeP2(v, 1, pTab->tnum); + sqlite3VdbeChangeP3(v, 1, flags); + sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); #endif /* Remove either the OP_OpenWrite or OpenRead. Set the P2 ** parameter of the other to pTab->tnum. */ - sqlite3VdbeChangeToNoop(v, 4 - flags); - sqlite3VdbeChangeP2(v, 3 + flags, pTab->tnum); - sqlite3VdbeChangeP3(v, 3 + flags, iDb); + sqlite3VdbeChangeToNoop(v, 3 - flags); + sqlite3VdbeChangeP2(v, 2 + flags, pTab->tnum); + sqlite3VdbeChangeP3(v, 2 + flags, iDb); /* Configure the number of columns. Configure the cursor to ** think that the table has one more column than it really @@ -73050,8 +74451,8 @@ SQLITE_API int sqlite3_blob_open( ** we can invoke OP_Column to fill in the vdbe cursors type ** and offset cache without causing any IO. */ - sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); - sqlite3VdbeChangeP2(v, 7, pTab->nCol); + sqlite3VdbeChangeP4(v, 2+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); + sqlite3VdbeChangeP2(v, 6, pTab->nCol); if( !db->mallocFailed ){ pParse->nVar = 1; pParse->nMem = 1; @@ -73577,7 +74978,6 @@ static int vdbeSorterIterInit( rc = sqlite3OsRead( pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); } if( rc==SQLITE_OK ){ @@ -73636,10 +75036,10 @@ static void vdbeSorterCompare( return; } } - r2->flags |= UNPACKED_PREFIX_MATCH; + assert( r2->default_rc==0 ); } - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } /* @@ -73731,23 +75131,40 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } } +/* +** Reset a sorting cursor back to its original empty state. +*/ +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ + if( pSorter->aIter ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(db, &pSorter->aIter[i]); + } + sqlite3DbFree(db, pSorter->aIter); + pSorter->aIter = 0; + } + if( pSorter->pTemp1 ){ + sqlite3OsCloseFree(pSorter->pTemp1); + pSorter->pTemp1 = 0; + } + vdbeSorterRecordFree(db, pSorter->pRecord); + pSorter->pRecord = 0; + pSorter->iWriteOff = 0; + pSorter->iReadOff = 0; + pSorter->nInMemory = 0; + pSorter->nTree = 0; + pSorter->nPMA = 0; + pSorter->aTree = 0; +} + + /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); - } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - } - vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; @@ -74183,14 +75600,55 @@ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, in if( pSorter->aTree ){ int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pCsr, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pSorter->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pSorter->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. */ + if( iRes<=0 ){ + pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); + pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); + pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + } }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; @@ -75276,6 +76734,8 @@ static int lookupName( }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ pExpr->iTable = 0; pTab = pParse->pTriggerTab; + }else{ + pTab = 0; } if( pTab ){ @@ -75319,8 +76779,8 @@ static int lookupName( /* ** Perhaps the name is a reference to the ROWID */ - assert( pTab!=0 || cntTab==0 ); - if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ + if( cnt==0 && cntTab==1 && pMatch && sqlite3IsRowid(zCol) + && HasRowid(pMatch->pTab) ){ cnt = 1; pExpr->iColumn = -1; /* IMP: R-44911-55124 */ pExpr->affinity = SQLITE_AFF_INTEGER; @@ -75648,7 +77108,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** likelihood(X, 0.0625). ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for ** likelihood(X,0.0625). */ - pExpr->iTable = 62; /* TUNING: Default 2nd arg to unlikely() is 0.0625 */ + /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ + pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; } } } @@ -76424,6 +77885,7 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ int op; pExpr = sqlite3ExprSkipCollate(pExpr); + if( pExpr->flags & EP_Generic ) return 0; op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -76456,7 +77918,11 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ ** If a memory allocation error occurs, that fact is recorded in pParse->db ** and the pExpr parameter is returned unchanged. */ -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){ +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName /* Name of collating sequence */ +){ if( pCollName->n>0 ){ Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); if( pNew ){ @@ -76509,6 +77975,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ Expr *p = pExpr; while( p ){ int op = p->op; + if( p->flags & EP_Generic ) break; if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; continue; @@ -77340,7 +78807,6 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags) if( p==0 ) return 0; pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; - pNew->iECursor = 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); @@ -77451,10 +78917,8 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->iLimit = 0; pNew->iOffset = 0; pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->pRightmost = 0; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); return pNew; @@ -77761,24 +79225,6 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ } } -/* -** Generate an OP_IsNull instruction that tests register iReg and jumps -** to location iDest if the value in iReg is NULL. The value in iReg -** was computed by pExpr. If we can look at pExpr at compile-time and -** determine that it can never generate a NULL, then the OP_IsNull operation -** can be omitted. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump( - Vdbe *v, /* The VDBE under construction */ - const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */ - int iReg, /* Test the value in this register for NULL */ - int iDest /* Jump here if the value is null */ -){ - if( sqlite3ExprCanBeNull(pExpr) ){ - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest); - } -} - /* ** Return TRUE if the given expression is a constant which would be ** unchanged by OP_Affinity with the affinity given in the second @@ -77975,7 +79421,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ pExpr = p->pEList->a[0].pExpr; iCol = (i16)pExpr->iColumn; - /* Code an OP_VerifyCookie and OP_TableLock for . */ + /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); @@ -77986,9 +79432,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ */ assert(v); if( iCol<0 ){ - int iAddr; - - iAddr = sqlite3CodeOnce(pParse); + int iAddr = sqlite3CodeOnce(pParse); + VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; @@ -78013,18 +79458,18 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq && (!mustBeUnique || (pIdx->nKeyCol==1 && pIdx->onError!=OE_None)) ){ - int iAddr = sqlite3CodeOnce(pParse); + int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - sqlite3VdbeJumpHere(v, iAddr); if( prNotFound && !pTab->aCol[iCol].notNull ){ *prNotFound = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); } + sqlite3VdbeJumpHere(v, iAddr); } } } @@ -78041,7 +79486,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ *prNotFound = rMayHaveNull = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ - testcase( pParse->nQueryLoop>0 ); pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; @@ -78113,7 +79557,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - testAddr = sqlite3CodeOnce(pParse); + testAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN @@ -78154,7 +79598,6 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( */ pExpr->iTable = pParse->nTab++; addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -78230,6 +79673,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( isRowid ){ sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); @@ -78291,7 +79735,7 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( testAddr>=0 ){ sqlite3VdbeJumpHere(v, testAddr); } - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); return rReg; } @@ -78353,10 +79797,11 @@ static void sqlite3ExprCodeIN( if( destIfNull==destIfFalse ){ /* Shortcut for the common case where the false and NULL outcomes are ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); sqlite3VdbeJumpHere(v, addr1); } @@ -78364,8 +79809,9 @@ static void sqlite3ExprCodeIN( if( eType==IN_INDEX_ROWID ){ /* In this case, the RHS is the ROWID of table b-tree */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + VdbeCoverage(v); }else{ /* In this case, the RHS is an index b-tree. */ @@ -78386,19 +79832,20 @@ static void sqlite3ExprCodeIN( ** for this particular IN operator. */ sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - + VdbeCoverage(v); }else{ /* In this branch, the RHS of the IN might contain a NULL and ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1, j2, j3; + int j1, j2; /* First check to see if the LHS is contained in the RHS. If so, ** then the presence of NULLs in the RHS does not matter, so jump ** over all of the code that follows. */ j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); /* Here we begin generating code that runs if the LHS is not ** contained within the RHS. Generate additional code that @@ -78406,18 +79853,15 @@ static void sqlite3ExprCodeIN( ** jump to destIfNull. If there are no NULLs in the RHS then ** jump to destIfFalse. */ - j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull); - j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); - sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull); - sqlite3VdbeJumpHere(v, j3); - sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1); - sqlite3VdbeJumpHere(v, j2); - - /* Jump to the appropriate target depending on whether or not - ** the RHS contains a NULL - */ - sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); + sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IfNot, rRhsHasNull, destIfFalse); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, rRhsHasNull); sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeAddOp2(v, OP_Integer, 1, rRhsHasNull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); /* The OP_Found at the top of this branch jumps here when true, ** causing the overall IN expression evaluation to fall through. @@ -78426,7 +79870,7 @@ static void sqlite3ExprCodeIN( } } sqlite3ReleaseTempReg(pParse, r1); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -78483,7 +79927,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } @@ -78493,7 +79937,14 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else - codeReal(v, z, negFlag, iMem); +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } #endif } } @@ -78609,15 +80060,14 @@ SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ /* ** Remove from the column cache any entries that were added since the -** the previous N Push operations. In other words, restore the cache -** to the state it was in N Pushes ago. +** the previous sqlite3ExprCachePush operation. In other words, restore +** the cache to the state it was in prior the most recent Push. */ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse, int N){ +SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ int i; struct yColCache *p; - assert( N>0 ); - assert( pParse->iCacheLevel>=N ); - pParse->iCacheLevel -= N; + assert( pParse->iCacheLevel>=1 ); + pParse->iCacheLevel--; #ifdef SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("POP to %d\n", pParse->iCacheLevel); @@ -78746,7 +80196,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int n int i; struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); for(i=0, p=pParse->aColCache; iiReg; if( x>=iFrom && xpLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -78967,6 +80411,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -78983,28 +80429,17 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_LSHIFT: case TK_RSHIFT: case TK_CONCAT: { - assert( TK_AND==OP_And ); - assert( TK_OR==OP_Or ); - assert( TK_PLUS==OP_Add ); - assert( TK_MINUS==OP_Subtract ); - assert( TK_REM==OP_Remainder ); - assert( TK_BITAND==OP_BitAnd ); - assert( TK_BITOR==OP_BitOr ); - assert( TK_SLASH==OP_Divide ); - assert( TK_LSHIFT==OP_ShiftLeft ); - assert( TK_RSHIFT==OP_ShiftRight ); - assert( TK_CONCAT==OP_Concat ); - testcase( op==TK_AND ); - testcase( op==TK_OR ); - testcase( op==TK_PLUS ); - testcase( op==TK_MINUS ); - testcase( op==TK_REM ); - testcase( op==TK_BITAND ); - testcase( op==TK_BITOR ); - testcase( op==TK_SLASH ); - testcase( op==TK_LSHIFT ); - testcase( op==TK_RSHIFT ); - testcase( op==TK_CONCAT ); + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); + assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); + assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); + assert( TK_BITAND==OP_BitAnd ); testcase( op==TK_BITAND ); + assert( TK_BITOR==OP_BitOr ); testcase( op==TK_BITOR ); + assert( TK_SLASH==OP_Divide ); testcase( op==TK_SLASH ); + assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); + assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); + assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); sqlite3VdbeAddOp3(v, op, r2, r1, target); @@ -79036,10 +80471,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } case TK_BITNOT: case TK_NOT: { - assert( TK_BITNOT==OP_BitNot ); - assert( TK_NOT==OP_Not ); - testcase( op==TK_BITNOT ); - testcase( op==TK_NOT ); + assert( TK_BITNOT==OP_BitNot ); testcase( op==TK_BITNOT ); + assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); inReg = target; @@ -79049,14 +80482,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_ISNULL: case TK_NOTNULL: { int addr; - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); sqlite3VdbeAddOp2(v, OP_Integer, 1, target); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); addr = sqlite3VdbeAddOp1(v, op, r1); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); sqlite3VdbeJumpHere(v, addr); break; @@ -79108,10 +80541,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); } sqlite3VdbeResolveLabel(v, endCoalesce); break; @@ -79163,9 +80597,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, + sqlite3ExprCodeExprList(pParse, pFarg, r1, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); - sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ + sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ r1 = 0; } @@ -79245,13 +80679,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) r3 = sqlite3GetTempReg(pParse); r4 = sqlite3GetTempReg(pParse); codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); + r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); pLItem++; pRight = pLItem->pExpr; sqlite3ReleaseTempReg(pParse, regFree2); r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); testcase( regFree2==0 ); codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); sqlite3ReleaseTempReg(pParse, r3); sqlite3ReleaseTempReg(pParse, r4); @@ -79384,13 +80819,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } if( (nExpr&1)!=0 ){ sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } @@ -79418,6 +80853,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( pExpr->affinity==OE_Ignore ){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + VdbeCoverage(v); }else{ sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, pExpr->affinity, pExpr->u.zToken, 0, 0); @@ -79505,7 +80941,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ ** results in register target. The results are guaranteed to appear ** in register target. */ -SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ +SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ int inReg; assert( target>0 && target<=pParse->nMem ); @@ -79518,7 +80954,20 @@ SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } } - return target; +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. If the expression is constant, then this routine +** might choose to code the expression at initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ + if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprCodeAtInit(pParse, pExpr, target, 0); + }else{ + sqlite3ExprCode(pParse, pExpr, target); + } } /* @@ -79533,25 +80982,16 @@ SQLITE_PRIVATE int sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ ** times. They are evaluated once and the results of the expression ** are reused. */ -SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ +SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ Vdbe *v = pParse->pVdbe; - int inReg; - inReg = sqlite3ExprCode(pParse, pExpr, target); + int iMem; + assert( target>0 ); - /* The only place, other than this routine, where expressions can be - ** converted to TK_REGISTER is internal subexpressions in BETWEEN and - ** CASE operators. Neither ever calls this routine. And this routine - ** is never called twice on the same expression. Hence it is impossible - ** for the input to this routine to already be a register. Nevertheless, - ** it seems prudent to keep the ALWAYS() in case the conditions above - ** change with future modifications or enhancements. */ - if( ALWAYS(pExpr->op!=TK_REGISTER) ){ - int iMem; - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, inReg, iMem); - exprToRegister(pExpr, iMem); - } - return inReg; + assert( pExpr->op!=TK_REGISTER ); + sqlite3ExprCode(pParse, pExpr, target); + iMem = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); + exprToRegister(pExpr, iMem); } #if defined(SQLITE_ENABLE_TREE_EXPLAIN) @@ -79964,7 +81404,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { @@ -79972,7 +81412,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -79986,23 +81426,17 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { - assert( TK_LT==OP_Lt ); - assert( TK_LE==OP_Le ); - assert( TK_GT==OP_Gt ); - assert( TK_GE==OP_Ge ); - assert( TK_EQ==OP_Eq ); - assert( TK_NE==OP_Ne ); - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -80016,18 +81450,20 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { - assert( TK_ISNULL==OP_IsNull ); - assert( TK_NOTNULL==OP_NotNull ); - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); sqlite3VdbeAddOp2(v, op, r1, dest); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); testcase( regFree1==0 ); break; } @@ -80054,6 +81490,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); testcase( regFree1==0 ); testcase( jumpIfNull==0 ); } @@ -80121,7 +81558,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { @@ -80131,7 +81568,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -80145,17 +81582,17 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { - testcase( op==TK_LT ); - testcase( op==TK_LE ); - testcase( op==TK_GT ); - testcase( op==TK_GE ); - testcase( op==TK_EQ ); - testcase( op==TK_NE ); testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -80169,16 +81606,18 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); + VdbeCoverageIf(v, op==TK_EQ); + VdbeCoverageIf(v, op==TK_NE); testcase( regFree1==0 ); testcase( regFree2==0 ); break; } case TK_ISNULL: case TK_NOTNULL: { - testcase( op==TK_ISNULL ); - testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); + testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); testcase( regFree1==0 ); break; } @@ -80207,6 +81646,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); testcase( regFree1==0 ); testcase( jumpIfNull==0 ); } @@ -80753,8 +82193,8 @@ static void renameTableFunc( assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } @@ -80792,6 +82232,7 @@ static void renameParentFunc( int token; /* Type of token */ UNUSED_PARAMETER(NotUsed); + if( zInput==0 || zOld==0 ) return; for(z=zInput; *z; z=z+n){ n = sqlite3GetToken(z, &token); if( token==TK_REFERENCES ){ @@ -80806,7 +82247,7 @@ static void renameParentFunc( sqlite3Dequote(zParent); if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), z-zInput, zInput, (const char *)zNew + (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew ); sqlite3DbFree(db, zOutput); zOutput = zOut; @@ -80892,8 +82333,8 @@ static void renameTriggerFunc( /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, - zTableName, tname.z+tname.n); + zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), + zSql, zTableName, tname.z+tname.n); sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); } } @@ -81145,7 +82586,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* Begin a transaction and code the VerifyCookie for database iDb. + /* Begin a transaction for database iDb. ** Then modify the schema cookie (since the ALTER TABLE modifies the ** schema). Open a statement transaction if the table is a virtual ** table. @@ -81281,6 +82722,7 @@ SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse *pParse, int iDb, int minForm sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp2(v, OP_Integer, minFormat, r2); j1 = sqlite3VdbeAddOp3(v, OP_Ge, r2, 0, r1); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, r2); sqlite3VdbeJumpHere(v, j1); sqlite3ReleaseTempReg(pParse, r1); @@ -81750,6 +83192,7 @@ static void openStatTable( assert( i1 ); /* >1 because it includes the rowid column */ + assert( nCol>0 ); nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); /* Allocate the space required for the Stat4Accum object */ n = sizeof(*p) @@ -81919,6 +83380,7 @@ static void statInit( p->db = db; p->nRow = 0; p->nCol = nCol; + p->nKeyCol = nKeyCol; p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; @@ -81929,9 +83391,9 @@ static void statInit( p->iGet = -1; p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[1])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; - p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[1])*0xd0944565; + p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; /* Set up the Stat4Accum.a[] and aBest[] arrays */ p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; @@ -81954,7 +83416,7 @@ static void statInit( sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); } static const FuncDef statInitFuncdef = { - 1+IsStat34, /* nArg */ + 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -82178,7 +83640,10 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** The SQL function always returns NULL. +** This SQL function always returns NULL. It's purpose it to accumulate +** statistical data and/or samples in the Stat4Accum object about the +** index being analyzed. The stat_get() SQL function will later be used to +** extract relevant information for constructing the sqlite_statN tables. ** ** The R parameter is only used for STAT3 and STAT4 */ @@ -82195,7 +83660,7 @@ static void statPush( UNUSED_PARAMETER( argc ); UNUSED_PARAMETER( context ); - assert( p->nCol>1 ); /* Includes rowid field */ + assert( p->nCol>0 ); assert( iChngnCol ); if( p->nRow==0 ){ @@ -82272,7 +83737,10 @@ static const FuncDef statPushFuncdef = { /* ** Implementation of the stat_get(P,J) SQL function. This routine is -** used to query the results. Content is returned for parameter J +** used to query statistical information that has been gathered into +** the Stat4Accum object by prior calls to stat_push(). The P parameter +** is a BLOB which is decoded into a pointer to the Stat4Accum objects. +** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** ** If neither STAT3 nor STAT4 are enabled, then J is always @@ -82323,7 +83791,7 @@ static void statGet( char *z; int i; - char *zRet = sqlite3MallocZero(p->nCol * 25); + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); if( zRet==0 ){ sqlite3_result_error_nomem(context); return; @@ -82331,7 +83799,7 @@ static void statGet( sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); z = zRet + sqlite3Strlen30(zRet); - for(i=0; i<(p->nCol-1); i++){ + for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; sqlite3_snprintf(24, z, " %llu", iVal); @@ -82491,27 +83959,27 @@ static void analyzeOneTable( sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol; /* Number of columns indexed by pIdx */ - int *aGotoChng; /* Array of jump instruction addresses */ + int nCol; /* Number of columns in pIdx. "N" */ int addrRewind; /* Address of "OP_Rewind iIdxCur" */ - int addrGotoChng0; /* Address of "Goto addr_chng_0" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; - VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); - nCol = pIdx->nKeyCol; - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); - if( aGotoChng==0 ) continue; - - /* Populate the register containing the index name. */ - if( pIdx->autoIndex==2 && !HasRowid(pTab) ){ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; zIdxName = pTab->zName; + nColTest = nCol - 1; }else{ + nCol = pIdx->nColumn; zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; } + + /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* ** Pseudo-code for loop that calls stat_push(): @@ -82536,7 +84004,7 @@ static void analyzeOneTable( ** regPrev(1) = idx(1) ** ... ** - ** chng_addr_N: + ** endDistinctTest: ** regRowid = idx(rowid) ** stat_push(P, regChng, regRowid) ** Next csr @@ -82549,7 +84017,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nCol); + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -82559,18 +84027,22 @@ static void analyzeOneTable( /* Invoke the stat_init() function. The arguments are: ** - ** (1) the number of columns in the index including the rowid, - ** (2) the number of rows in the index, + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) the number of rows in the index, + ** ** - ** The second argument is only used for STAT3 and STAT4 + ** The third argument is only used for STAT3 and STAT4 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); #endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1+IsStat34); + sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: ** @@ -82581,44 +84053,64 @@ static void analyzeOneTable( ** */ addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); - addrGotoChng0 = sqlite3VdbeAddOp0(v, OP_Goto); - - /* - ** next_row: - ** regChng = 0 - ** if( idx(0) != regPrev(0) ) goto chng_addr_0 - ** regChng = 1 - ** if( idx(1) != regPrev(1) ) goto chng_addr_1 - ** ... - ** regChng = N - ** goto chng_addr_N - */ addrNextRow = sqlite3VdbeCurrentAddr(v); - for(i=0; iazColl[i]); - sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); - aGotoChng[i] = - sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - } - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regChng); - aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto); - /* - ** chng_addr_0: - ** regPrev(0) = idx(0) - ** chng_addr_1: - ** regPrev(1) = idx(1) - ** ... - */ - sqlite3VdbeJumpHere(v, addrGotoChng0); - for(i=0; i0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(v); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && pIdx->onError!=OE_None ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; inMem = MAX(pParse->nMem, regCol+nCol+1); + pParse->nMem = MAX(pParse->nMem, regCol+nCol); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + VdbeCoverage(v); callStatGet(v, regStat4, STAT_GET_NEQ, regEq); callStatGet(v, regStat4, STAT_GET_NLT, regLt); callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); + /* We know that the regSampleRowid row exists because it was read by + ** the previous loop. Thus the not-found jump of seekOp will never + ** be taken */ + VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pIdx->aiColumn[0], regSample); @@ -82687,19 +84183,18 @@ static void analyzeOneTable( i16 iCol = pIdx->aiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regTemp, "bbbbbb", 0); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeAddOp2(v, OP_Goto, 1, addrNext); /* P1==1 for end-of-loop */ sqlite3VdbeJumpHere(v, addrIsNull); } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* End of analysis */ sqlite3VdbeJumpHere(v, addrRewind); - sqlite3DbFree(db, aGotoChng); } @@ -82709,7 +84204,7 @@ static void analyzeOneTable( if( pOnlyIdx==0 && needTableCnt ){ VdbeComment((v, "%s", pTab->zName)); sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); @@ -82800,6 +84295,7 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -82847,6 +84343,8 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -82868,6 +84366,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -82886,7 +84385,17 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( aOut ){ + aOut[i] = v; + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -82894,14 +84403,19 @@ static void decodeIntArray( #else if( pIndex ) #endif - { - if( strcmp(z, "unordered")==0 ){ + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - int v32 = 0; - sqlite3GetInt32(z+3, &v32); - pIndex->szIdxRow = sqlite3LogEst(v32); + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; } } @@ -82942,12 +84456,16 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + pIndex->bUnordered = 0; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -82988,7 +84506,16 @@ static void initAvgEq(Index *pIdx){ IndexSample *aSample = pIdx->aSample; IndexSample *pFinal = &aSample[pIdx->nSample-1]; int iCol; - for(iCol=0; iColnKeyCol; iCol++){ + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColaAvgEq[iCol] = avgEq; - if( pIdx->nSampleCol==1 ) break; } } } @@ -83070,7 +84596,6 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - int nAvgCol = 1; /* Number of entries in Index.aAvgEq */ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ @@ -83088,13 +84613,17 @@ static int loadStatTbl( ** loaded from the stat4 table. In this case ignore stat3 data. */ if( pIdx==0 || pIdx->nSample ) continue; if( bStat3==0 ){ - nIdxCol = pIdx->nKeyCol+1; - nAvgCol = pIdx->nKeyCol; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } } pIdx->nSampleCol = nIdxCol; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; - nByte += nAvgCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ @@ -83102,7 +84631,7 @@ static int loadStatTbl( return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; - pIdx->aAvgEq = pSpace; pSpace += nAvgCol; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; @@ -83139,9 +84668,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the @@ -84211,6 +85740,19 @@ static void codeTableLocks(Parse *pParse){ #define codeTableLocks(x) #endif +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; icookieGoto>0 ){ - yDbMask mask; - int iDb, i, addr; - sqlite3VdbeJumpHere(v, pParse->cookieGoto-1); - for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ - if( (mask & pParse->cookieMask)==0 ) continue; + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ + int iDb, i; + assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + for(iDb=0; iDbnDb; iDb++){ + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v,OP_Transaction, iDb, (mask & pParse->writeMask)!=0); - if( db->init.busy==0 ){ - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp3(v, OP_VerifyCookie, - iDb, pParse->cookieValue[iDb], - db->aDb[iDb].pSchema->iGeneration); - } + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pParse->cookieValue[iDb], /* P3 */ + db->aDb[iDb].pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ @@ -84281,17 +85826,16 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3AutoincrementBegin(pParse); /* Code constant expressions that where factored out of inner loops */ - addr = pParse->cookieGoto; if( pParse->pConstExpr ){ ExprList *pEL = pParse->pConstExpr; - pParse->cookieGoto = 0; + pParse->okConstFactor = 0; for(i=0; inExpr; i++){ sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); } } /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, 1); } } @@ -84313,8 +85857,7 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; - pParse->cookieMask = 0; - pParse->cookieGoto = 0; + DbMaskZero(pParse->cookieMask); } /* @@ -84855,7 +86398,7 @@ SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ */ SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ Index *p; - for(p=pTab->pIndex; p && p->autoIndex!=2; p=p->pNext){} + for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} return p; } @@ -85003,7 +86546,7 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -85046,7 +86589,7 @@ SQLITE_PRIVATE void sqlite3StartTable( reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); + j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); @@ -85384,7 +86927,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, 0, sortOrder, 0); if( p ){ - p->autoIndex = 2; + p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; if( v ) sqlite3VdbeJumpHere(v, pParse->addrSkipPK); } pList = 0; @@ -85404,7 +86947,10 @@ SQLITE_PRIVATE void sqlite3AddCheckConstraint( ){ #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; - if( pTab && !IN_DECLARE_VTAB ){ + sqlite3 *db = pParse->db; + if( pTab && !IN_DECLARE_VTAB + && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) + ){ pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); if( pParse->constraintName.n ){ sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); @@ -85756,7 +87302,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ assert( pParse->pNewTable==pTab ); pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); if( pPk==0 ) return; - pPk->autoIndex = 2; + pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); @@ -85779,7 +87325,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int n; - if( pIdx->autoIndex==2 ) continue; + if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; } @@ -86773,27 +88319,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); - sqlite3VdbeResolveLabel(v, iPartIdxLabel); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, (char *)pKey, P4_KEYINFO); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); - addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); assert( pKey!=0 || db->mallocFailed || pParse->nErr ); if( pIndex->onError!=OE_None && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, - pKey->nField - pIndex->nKeyCol); + pKey->nField - pIndex->nKeyCol); VdbeCoverage(v); sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); }else{ addr2 = sqlite3VdbeCurrentAddr(v); @@ -86802,7 +88348,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); @@ -86828,15 +88374,15 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -86859,7 +88405,7 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( ** ** If the index is created successfully, return a pointer to the new Index ** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.autoIndex==2). +** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ SQLITE_PRIVATE Index *sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ @@ -87066,7 +88612,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -87074,7 +88620,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->autoIndex = (u8)(pName==0); + pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -87186,7 +88732,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int k; assert( pIdx->onError!=OE_None ); - assert( pIdx->autoIndex ); + assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); assert( pIndex->onError!=OE_None ); if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; @@ -87347,7 +88893,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -87358,20 +88904,27 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** are based on typical values found in actual indices. */ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ - tRowcnt *a = pIdx->aiRowEst; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ + a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* @@ -87402,7 +88955,7 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists pParse->checkSchema = 1; goto exit_drop_index; } - if( pIndex->autoIndex ){ + if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){ sqlite3ErrorMsg(pParse, "index associated with UNIQUE " "or PRIMARY KEY constraint cannot be dropped", 0); goto exit_drop_index; @@ -87571,7 +89124,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( assert( iStart<=pSrc->nSrc ); /* Allocate additional space if needed */ - if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; int nAlloc = pSrc->nSrc+nExtra; int nGot; @@ -87583,7 +89136,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( } pSrc = pNew; nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = (u8)nGot; + pSrc->nAlloc = nGot; } /* Move existing slots that come after the newly inserted slots @@ -87591,7 +89144,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( for(i=pSrc->nSrc-1; i>=iStart; i--){ pSrc->a[i+nExtra] = pSrc->a[i]; } - pSrc->nSrc += (i8)nExtra; + pSrc->nSrc += nExtra; /* Zero the newly allocated slots */ memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); @@ -87923,59 +89476,24 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ } /* -** Generate VDBE code that will verify the schema cookie and start -** a read-transaction for all named database files. -** -** It is important that all schema cookies be verified and all -** read transactions be started before anything else happens in -** the VDBE program. But this routine can be called after much other -** code has been generated. So here is what we do: -** -** The first time this routine is called, we code an OP_Goto that -** will jump to a subroutine at the end of the program. Then we -** record every database that needs its schema verified in the -** pParse->cookieMask field. Later, after all other code has been -** generated, the subroutine that does the cookie verifications and -** starts the transactions will be coded and the OP_Goto P2 value -** will be made to point to that subroutine. The generation of the -** cookie verification subroutine code happens in sqlite3FinishCoding(). -** -** If iDb<0 then code the OP_Goto only - don't set flag to verify the -** schema on any databases. This can be used to position the OP_Goto -** early in the code, before we know if any database tables will be used. +** Record the fact that the schema cookie will need to be verified +** for database iDb. The code to actually verify the schema cookie +** will occur at the end of the top-level VDBE and will be generated +** later, by sqlite3FinishCoding(). */ SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3 *db = pToplevel->db; -#ifndef SQLITE_OMIT_TRIGGER - if( pToplevel!=pParse ){ - /* This branch is taken if a trigger is currently being coded. In this - ** case, set cookieGoto to a non-zero value to show that this function - ** has been called. This is used by the sqlite3ExprCodeConstants() - ** function. */ - pParse->cookieGoto = -1; - } -#endif - if( pToplevel->cookieGoto==0 ){ - Vdbe *v = sqlite3GetVdbe(pToplevel); - if( v==0 ) return; /* This only happens if there was a prior error */ - pToplevel->cookieGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0)+1; - } - if( iDb>=0 ){ - sqlite3 *db = pToplevel->db; - yDbMask mask; - - assert( iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); - assert( iDbcookieMask & mask)==0 ){ - pToplevel->cookieMask |= mask; - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; - if( !OMIT_TEMPDB && iDb==1 ){ - sqlite3OpenTempDatabase(pToplevel); - } + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbcookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); + pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); } } } @@ -88011,7 +89529,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3CodeVerifySchema(pParse, iDb); - pToplevel->writeMask |= ((yDbMask)1)<writeMask, iDb); pToplevel->isMultiWrite |= setStatement; } @@ -88094,7 +89612,8 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, - (pIdx->autoIndex==2)?SQLITE_CONSTRAINT_PRIMARYKEY:SQLITE_CONSTRAINT_UNIQUE, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + : SQLITE_CONSTRAINT_UNIQUE, onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); } @@ -88946,10 +90465,8 @@ SQLITE_PRIVATE void sqlite3MaterializeView( SrcList *pFrom; sqlite3 *db = pParse->db; int iDb = sqlite3SchemaToIndex(db, pView->pSchema); - pWhere = sqlite3ExprDup(db, pWhere, 0); pFrom = sqlite3SrcListAppend(db, 0, 0, 0); - if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); @@ -88957,10 +90474,7 @@ SQLITE_PRIVATE void sqlite3MaterializeView( assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0); - if( pSel ) pSel->selFlags |= SF_Materialize; - sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -89297,7 +90811,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( iKey = ++pParse->nMem; nKey = 0; /* Zero tells OP_Found to use a composite key */ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(v, pPk), P4_TRANSIENT); + sqlite3IndexAffinityStr(v, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ /* Get the rowid of the row to be deleted and remember it in the RowSet */ @@ -89335,13 +90849,15 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); + VdbeCoverage(v); } }else if( pPk ){ - addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); + addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey); assert( nKey==0 ); /* OP_Found will use a composite key */ }else{ addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); + VdbeCoverage(v); assert( nKey==1 ); } @@ -89365,7 +90881,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( okOnePass ){ sqlite3VdbeResolveLabel(v, addrBypass); }else if( pPk ){ - sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); + sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); }else{ sqlite3VdbeAddOp2(v, OP_Goto, 0, addrLoop); @@ -89463,7 +90979,11 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ) sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + if( !bNoSeek ){ + sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + VdbeCoverageIf(v, opSeek==OP_NotExists); + VdbeCoverageIf(v, opSeek==OP_NotFound); + } /* If there are any triggers to fire, allocate a range of registers to ** use for the old.* references in the triggers. */ @@ -89505,6 +91025,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( */ if( addrStartuniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); - sqlite3VdbeResolveLabel(v, iPartIdxLabel); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } } @@ -89604,10 +91126,11 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( ** ** If *piPartIdxLabel is not NULL, fill it in with a label and jump ** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). ** A partial index should be skipped if its WHERE clause evaluates ** to false or null. If pIdx is not a partial index, *piPartIdxLabel ** will be set to zero which is an empty label that is ignored by -** sqlite3VdbeResolveLabel(). +** sqlite3ResolvePartIdxLabel(). ** ** The pPrior and regPrior parameters are used to implement a cache to ** avoid unnecessary register loads. If pPrior is not NULL, then it is @@ -89640,6 +91163,7 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); pParse->iPartIdxTab = iDataCur; + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); }else{ @@ -89668,6 +91192,18 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( return regBase; } +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + sqlite3ExprCachePop(pParse); + } +} + /************** End of delete.c **********************************************/ /************** Begin file func.c ********************************************/ /* @@ -90687,7 +92223,7 @@ static void charFunc( ){ unsigned char *z, *zOut; int i; - zOut = z = sqlite3_malloc( argc*4 ); + zOut = z = sqlite3_malloc( argc*4+1 ); if( z==0 ){ sqlite3_result_error_nomem(context); return; @@ -91211,7 +92747,7 @@ static void groupConcatStep( } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( nVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); } } static void groupConcatFinalize(sqlite3_context *context){ @@ -91361,6 +92897,7 @@ SQLITE_PRIVATE void sqlite3RegisterGlobalFunctions(void){ FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), @@ -91655,8 +93192,8 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( if( zKey==0 ){ /* If zKey is NULL, then this foreign key is implicitly mapped to ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be - ** identified by the test (Index.autoIndex==2). */ - if( pIdx->autoIndex==2 ){ + ** identified by the test. */ + if( IsPrimaryKeyIndex(pIdx) ){ if( aiCol ){ int i; for(i=0; iaCol[i].iFrom; @@ -91762,10 +93299,11 @@ static void fkLookupParent( ** search for a matching row in the parent table. */ if( nIncr<0 ){ sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + VdbeCoverage(v); } for(i=0; inCol; i++){ int iReg = aiCol[i] + regData + 1; - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); } if( isIgnore==0 ){ @@ -91782,17 +93320,19 @@ static void fkLookupParent( ** will have INTEGER affinity applied to it, which may not be correct. */ sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + VdbeCoverage(v); /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not ** increment the constraint-counter. */ if( pTab==pFKey->pFrom && nIncr==1 ){ - sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); } sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); sqlite3VdbeJumpHere(v, iMustBeInt); @@ -91828,15 +93368,15 @@ static void fkLookupParent( /* The parent key is a composite key that includes the IPK column */ iParent = regData; } - sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); - sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + sqlite3IndexAffinityStr(v,pIdx), nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempRange(pParse, regTemp, nCol); @@ -91974,6 +93514,7 @@ static void fkScanChildren( if( nIncr<0 ){ iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); + VdbeCoverage(v); } /* Create an Expr object representing an SQL expression like: @@ -92136,7 +93677,7 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa } if( !p ) return; iSkip = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); } pParse->disableTriggers = 1; @@ -92154,6 +93695,7 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa */ if( (db->flags & SQLITE_DeferFKs)==0 ){ sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, OE_Abort, 0, P4_STATIC, P5_ConstraintFK); } @@ -92313,7 +93855,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; for(i=0; inCol; i++){ int iReg = pFKey->aCol[i].iFrom + regOld + 1; - sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); } @@ -92880,10 +94422,16 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ } /* -** Set P4 of the most recently inserted opcode to a column affinity -** string for table pTab. A column affinity string has one character -** for each column indexed by the index, according to the affinity of the -** column: +** Compute the affinity string for table pTab, if it has not already been +** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. +** +** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and +** if iReg>0 then code an OP_Affinity opcode that will set the affinities +** for register iReg and following. Or if affinities exists and iReg==0, +** then just set the P4 operand of the previous opcode (which should be +** an OP_MakeRecord) to the affinity string. +** +** A column affinity string has one character per column: ** ** Character Column affinity ** ------------------------------ @@ -92893,19 +94441,11 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** 'd' INTEGER ** 'e' REAL */ -SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ - /* The first time a column affinity string for a particular table - ** is required, it is allocated and populated here. It is then - ** stored as a member of the Table structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqlite3DeleteTable() when the Table structure itself is cleaned up. - */ - if( !pTab->zColAff ){ - char *zColAff; - int i; +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ + int i; + char *zColAff = pTab->zColAff; + if( zColAff==0 ){ sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ db->mallocFailed = 1; @@ -92915,22 +94455,28 @@ SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *v, Table *pTab){ for(i=0; inCol; i++){ zColAff[i] = pTab->aCol[i].affinity; } - zColAff[pTab->nCol] = '\0'; - + do{ + zColAff[i--] = 0; + }while( i>=0 && zColAff[i]==SQLITE_AFF_NONE ); pTab->zColAff = zColAff; } - - sqlite3VdbeChangeP4(v, -1, pTab->zColAff, P4_TRANSIENT); + i = sqlite3Strlen30(zColAff); + if( i ){ + if( iReg ){ + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); + }else{ + sqlite3VdbeChangeP4(v, -1, zColAff, i); + } + } } /* ** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program beginning at location -** iStartAddr throught the end of the program. This is used to see if +** have been opened at any point in the VDBE program. This is used to see if ** a statement of the form "INSERT INTO SELECT ..." can -** run without using temporary table for the results of the SELECT. +** run without using a temporary table for the results of the SELECT. */ -static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ +static int readsTable(Parse *p, int iDb, Table *pTab){ Vdbe *v = sqlite3GetVdbe(p); int i; int iEnd = sqlite3VdbeCurrentAddr(v); @@ -92938,7 +94484,7 @@ static int readsTable(Parse *p, int iStartAddr, int iDb, Table *pTab){ VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; #endif - for(i=iStartAddr; iopcode==OP_OpenRead && pOp->p3==iDb ){ @@ -93039,14 +94585,14 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); addr = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4(v, OP_String8, 0, memId-1, 0, p->pTab->zName, 0); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); + sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); + sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); sqlite3VdbeAddOp2(v, OP_Goto, 0, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); + sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); sqlite3VdbeAddOp0(v, OP_Close); } @@ -93081,25 +94627,16 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ Db *pDb = &db->aDb[p->iDb]; - int j1, j2, j3, j4, j5; + int j1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); - j2 = sqlite3VdbeAddOp0(v, OP_Rewind); - j3 = sqlite3VdbeAddOp3(v, OP_Column, 0, 0, iRec); - j4 = sqlite3VdbeAddOp3(v, OP_Eq, memId-1, 0, iRec); - sqlite3VdbeAddOp2(v, OP_Next, 0, j3); - sqlite3VdbeJumpHere(v, j2); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - j5 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, j4); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeJumpHere(v, j5); sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -93117,97 +94654,6 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* -** Generate code for a co-routine that will evaluate a subquery one -** row at a time. -** -** The pSelect parameter is the subquery that the co-routine will evaluation. -** Information about the location of co-routine and the registers it will use -** is returned by filling in the pDest object. -** -** Registers are allocated as follows: -** -** pDest->iSDParm The register holding the next entry-point of the -** co-routine. Run the co-routine to its next breakpoint -** by calling "OP_Yield $X" where $X is pDest->iSDParm. -** -** pDest->iSDParm+1 The register holding the "completed" flag for the -** co-routine. This register is 0 if the previous Yield -** generated a new result row, or 1 if the subquery -** has completed. If the Yield is called again -** after this register becomes 1, then the VDBE will -** halt with an SQLITE_INTERNAL error. -** -** pDest->iSdst First result register. -** -** pDest->nSdst Number of result registers. -** -** This routine handles all of the register allocation and fills in the -** pDest structure appropriately. -** -** Here is a schematic of the generated code assuming that X is the -** co-routine entry-point register reg[pDest->iSDParm], that EOF is the -** completed flag reg[pDest->iSDParm+1], and R and S are the range of -** registers that hold the result set, reg[pDest->iSdst] through -** reg[pDest->iSdst+pDest->nSdst-1]: -** -** X <- A -** EOF <- 0 -** goto B -** A: setup for the SELECT -** loop rows in the SELECT -** load results into registers R..S -** yield X -** end loop -** cleanup after the SELECT -** EOF <- 1 -** yield X -** halt-error -** B: -** -** To use this subroutine, the caller generates code as follows: -** -** [ Co-routine generated by this subroutine, shown above ] -** S: yield X -** if EOF goto E -** if skip this row, goto C -** if terminate loop, goto E -** deal with this row -** C: goto S -** E: -*/ -SQLITE_PRIVATE int sqlite3CodeCoroutine(Parse *pParse, Select *pSelect, SelectDest *pDest){ - int regYield; /* Register holding co-routine entry-point */ - int regEof; /* Register holding co-routine completion flag */ - int addrTop; /* Top of the co-routine */ - int j1; /* Jump instruction */ - int rc; /* Result code */ - Vdbe *v; /* VDBE under construction */ - - regYield = ++pParse->nMem; - regEof = ++pParse->nMem; - v = sqlite3GetVdbe(pParse); - addrTop = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_Integer, addrTop+2, regYield); /* X <- A */ - VdbeComment((v, "Co-routine entry point")); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEof); /* EOF <- 0 */ - VdbeComment((v, "Co-routine completion flag")); - sqlite3SelectDestInit(pDest, SRT_Coroutine, regYield); - j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); - rc = sqlite3Select(pParse, pSelect, pDest); - assert( pParse->nErr==0 || rc ); - if( pParse->db->mallocFailed && rc==SQLITE_OK ) rc = SQLITE_NOMEM; - if( rc ) return rc; - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, regYield); /* yield X */ - sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); - VdbeComment((v, "End of coroutine")); - sqlite3VdbeJumpHere(v, j1); /* label B: */ - return rc; -} - - - /* Forward declaration */ static int xferOptimization( Parse *pParse, /* Parser context */ @@ -93270,7 +94716,6 @@ static int xferOptimization( ** and the SELECT clause does not read from
    at any time. ** The generated code follows this template: ** -** EOF <- 0 ** X <- A ** goto B ** A: setup for the SELECT @@ -93279,12 +94724,9 @@ static int xferOptimization( ** yield X ** end loop ** cleanup after the SELECT -** EOF <- 1 -** yield X -** goto A +** end-coroutine X ** B: open write cursor to
    and its indices -** C: yield X -** if EOF goto D +** C: yield X, at EOF goto D ** insert the select result into
    from R..R+n ** goto C ** D: cleanup @@ -93295,7 +94737,6 @@ static int xferOptimization( ** we have to use a intermediate table to store the results of ** the select. The template is like this: ** -** EOF <- 0 ** X <- A ** goto B ** A: setup for the SELECT @@ -93304,12 +94745,9 @@ static int xferOptimization( ** yield X ** end loop ** cleanup after the SELECT -** EOF <- 1 -** yield X -** halt-error +** end co-routine R ** B: open temp table -** L: yield X -** if EOF goto M +** L: yield X, at EOF goto M ** insert row from R..R+n into temp table ** goto L ** M: open write cursor to
    and its indices @@ -93339,16 +94777,16 @@ SQLITE_PRIVATE void sqlite3Insert( int iIdxCur = 0; /* First index cursor */ int ipkColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ int endOfLoop; /* Label for the end of the insertion loop */ - int useTempTable = 0; /* Store SELECT results in intermediate table */ int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ int addrInsTop = 0; /* Jump to label "D" */ int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ - int addrSelect = 0; /* Address of coroutine that implements the SELECT */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ - int appendFlag = 0; /* True if the insert is likely to be an append */ - int withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 useTempTable = 0; /* Store SELECT results in intermediate table */ + u8 appendFlag = 0; /* True if the insert is likely to be an append */ + u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 bIdListInOrder = 1; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ @@ -93358,7 +94796,6 @@ SQLITE_PRIVATE void sqlite3Insert( int regIns; /* Block of regs holding rowid+data being inserted */ int regRowid; /* registers holding insert rowid */ int regData; /* register holding first column to insert */ - int regEof = 0; /* Register recording end of SELECT data */ int *aRegIdx = 0; /* One register allocated to each index */ #ifndef SQLITE_OMIT_TRIGGER @@ -93461,6 +94898,57 @@ SQLITE_PRIVATE void sqlite3Insert( */ regAutoinc = autoIncBegin(pParse, iDb, pTab); + /* Allocate registers for holding the rowid of the new row, + ** the content of the new row, and the assemblied row record. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the ipkColumn variable + ** the index into IDLIST of the primary key column. ipkColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the INTEGER + ** PRIMARY KEY in the original table is pTab->iPKey.) + */ + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; + } + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ + pColumn->a[i].idx = j; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ + ipkColumn = i; + bIdListInOrder = 0; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList, 0, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } + } + } + /* Figure out how many columns of data are supplied. If the data ** is coming from a SELECT statement, then generate a co-routine that ** produces a single row of the SELECT on each invocation. The @@ -93468,14 +94956,24 @@ SQLITE_PRIVATE void sqlite3Insert( */ if( pSelect ){ /* Data is coming from a SELECT. Generate a co-routine to run the SELECT */ - int rc = sqlite3CodeCoroutine(pParse, pSelect, &dest); - if( rc ) goto insert_cleanup; - - regEof = dest.iSDParm + 1; + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int rc; /* Result code */ + + regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); regFromSelect = dest.iSdst; + assert( pParse->nErr==0 || rc ); + if( rc || db->mallocFailed ) goto insert_cleanup; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; - assert( dest.nSdst==nColumn ); /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -93486,7 +94984,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** of the tables being read by the SELECT statement. Also use a ** temp table in the case of row triggers. */ - if( pTrigger || readsTable(pParse, addrSelect, iDb, pTab) ){ + if( pTrigger || readsTable(pParse, iDb, pTab) ){ useTempTable = 1; } @@ -93496,28 +94994,25 @@ SQLITE_PRIVATE void sqlite3Insert( ** here is from the 4th template: ** ** B: open temp table - ** L: yield X - ** if EOF goto M + ** L: yield X, goto M at EOF ** insert row from R..R+n into temp table ** goto L ** M: ... */ int regRec; /* Register to hold packed record */ int regTempRowid; /* Register to hold temp table ROWID */ - int addrTop; /* Label "L" */ - int addrIf; /* Address of jump to M */ + int addrL; /* Label "L" */ srcTab = pParse->nTab++; regRec = sqlite3GetTempReg(pParse); regTempRowid = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); - addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); - addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); + addrL = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - sqlite3VdbeJumpHere(v, addrIf); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrL); + sqlite3VdbeJumpHere(v, addrL); sqlite3ReleaseTempReg(pParse, regRec); sqlite3ReleaseTempReg(pParse, regTempRowid); } @@ -93538,6 +95033,14 @@ SQLITE_PRIVATE void sqlite3Insert( } } + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the ipkColumn variable to the integer primary key + ** column index in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + ipkColumn = pTab->iPKey; + } + /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ @@ -93556,52 +95059,6 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); goto insert_cleanup; } - - /* If the INSERT statement included an IDLIST term, then make sure - ** all elements of the IDLIST really are columns of the table and - ** remember the column indices. - ** - ** If the table has an INTEGER PRIMARY KEY column and that column - ** is named in the IDLIST, then record in the ipkColumn variable - ** the index into IDLIST of the primary key column. ipkColumn is - ** the index of the primary key as it appears in IDLIST, not as - ** is appears in the original table. (The index of the INTEGER - ** PRIMARY KEY in the original table is pTab->iPKey.) - */ - if( pColumn ){ - for(i=0; inId; i++){ - pColumn->a[i].idx = -1; - } - for(i=0; inId; i++){ - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ - pColumn->a[i].idx = j; - if( j==pTab->iPKey ){ - ipkColumn = i; assert( !withoutRowid ); - } - break; - } - } - if( j>=pTab->nCol ){ - if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ - ipkColumn = i; - }else{ - sqlite3ErrorMsg(pParse, "table %S has no column named %s", - pTabList, 0, pColumn->a[i].zName); - pParse->checkSchema = 1; - goto insert_cleanup; - } - } - } - } - - /* If there is no IDLIST term but the table has an integer primary - ** key, the set the ipkColumn variable to the integer primary key - ** column index in the original table definition. - */ - if( pColumn==0 && nColumn>0 ){ - ipkColumn = pTab->iPKey; - } /* Initialize the count of rows to be inserted */ @@ -93629,38 +95086,26 @@ SQLITE_PRIVATE void sqlite3Insert( /* This block codes the top of loop only. The complete loop is the ** following pseudocode (template 4): ** - ** rewind temp table + ** rewind temp table, if empty goto D ** C: loop over rows of intermediate table ** transfer values form intermediate table into
    ** end loop ** D: ... */ - addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); VdbeCoverage(v); addrCont = sqlite3VdbeCurrentAddr(v); }else if( pSelect ){ /* This block codes the top of loop only. The complete loop is the ** following pseudocode (template 3): ** - ** C: yield X - ** if EOF goto D + ** C: yield X, at EOF goto D ** insert the select result into
    from R..R+n ** goto C ** D: ... */ - addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); - addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); - } - - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. - */ - regRowid = regIns = pParse->nMem+1; - pParse->nMem += pTab->nCol + 1; - if( IsVirtual(pTab) ){ - regRowid++; - pParse->nMem++; + addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); } - regData = regRowid+1; /* Run the BEFORE and INSTEAD OF triggers, if there are any */ @@ -93685,10 +95130,10 @@ SQLITE_PRIVATE void sqlite3Insert( assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } /* Cannot have triggers on a virtual table. If it were possible, @@ -93722,8 +95167,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** table column affinities. */ if( !isView ){ - sqlite3VdbeAddOp2(v, OP_Affinity, regCols+1, pTab->nCol); - sqlite3TableAffinityStr(v, pTab); + sqlite3TableAffinity(v, pTab, regCols+1); } /* Fire BEFORE or INSTEAD OF triggers */ @@ -93745,7 +95189,7 @@ SQLITE_PRIVATE void sqlite3Insert( if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+ipkColumn, regRowid); + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); }else{ VdbeOp *pOp; sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); @@ -93764,14 +95208,14 @@ SQLITE_PRIVATE void sqlite3Insert( if( !appendFlag ){ int j1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); sqlite3VdbeJumpHere(v, j1); }else{ j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); } - sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } }else if( IsVirtual(pTab) || withoutRowid ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); @@ -93791,8 +95235,9 @@ SQLITE_PRIVATE void sqlite3Insert( /* The value of the INTEGER PRIMARY KEY column is always a NULL. ** Whenever this column is read, the rowid will be substituted ** in its place. Hence, fill this column with a NULL to avoid - ** taking up data space with information that will never be used. */ - sqlite3VdbeAddOp2(v, OP_Null, 0, iRegStore); + ** taking up data space with information that will never be used. + ** As there may be shallow copies of this value, make it a soft-NULL */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); continue; } if( pColumn==0 ){ @@ -93809,11 +95254,13 @@ SQLITE_PRIVATE void sqlite3Insert( } } if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, iRegStore); + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); }else if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); + } }else{ sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); } @@ -93859,7 +95306,7 @@ SQLITE_PRIVATE void sqlite3Insert( */ sqlite3VdbeResolveLabel(v, endOfLoop); if( useTempTable ){ - sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrInsTop); sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ @@ -94026,6 +95473,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ipkTop = 0; /* Top of the rowid change constraint check */ int ipkBottom = 0; /* Bottom of the rowid change constraint check */ u8 isUpdate; /* True if this is an UPDATE operation */ + u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ int regRowid = -1; /* Register holding ROWID value */ isUpdate = regOldData!=0; @@ -94080,15 +95528,17 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3VdbeAddOp4(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, regNewData+1+i, zMsg, P4_DYNAMIC); sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); break; } case OE_Ignore: { sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); + VdbeCoverage(v); break; } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); sqlite3VdbeJumpHere(v, j1); break; @@ -94140,6 +95590,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** it might have changed. Skip the conflict logic below if the rowid ** is unchanged. */ sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); } /* If the response to a rowid conflict is REPLACE but the response @@ -94159,6 +95611,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Check to see if the new rowid already exists in the table. Skip ** the following conflict logic if it does not. */ sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); + VdbeCoverage(v); /* Generate code that deals with a rowid collision */ switch( onError ){ @@ -94237,6 +95690,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } iThisCur = iIdxCur+ix; addrUniqueOk = sqlite3VdbeMakeLabel(v); @@ -94267,7 +95724,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), P4_TRANSIENT); VdbeComment((v, "for %s", pIdx->zName)); sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn); @@ -94295,7 +95751,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Check to see if the new index entry will be unique */ sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, - regIdx, pIdx->nKeyCol); + regIdx, pIdx->nKeyCol); VdbeCoverage(v); /* Generate code to handle collisions */ regR = (pIdx==pPk) ? regIdx : sqlite3GetTempRange(pParse, nPkField); @@ -94306,6 +95762,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** is different from old-rowid */ if( isUpdate ){ sqlite3VdbeAddOp3(v, OP_Eq, regR, addrUniqueOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); } }else{ int x; @@ -94329,7 +95787,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** KEY values of this row before the update. */ int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; int op = OP_Ne; - int regCmp = (pIdx->autoIndex==2 ? regIdx : regR); + int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); @@ -94341,6 +95799,9 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3VdbeAddOp4(v, op, regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ ); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverageIf(v, op==OP_Eq); + VdbeCoverageIf(v, op==OP_Ne); } } } @@ -94412,19 +95873,22 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( int regData; /* Content registers (after the rowid) */ int regRec; /* Register holding assemblied record for the table */ int i; /* Loop counter */ + u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ if( aRegIdx[i]==0 ) continue; + bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]); pik_flags = 0; if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; - if( pIdx->autoIndex==2 && !HasRowid(pTab) ){ + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; } @@ -94434,7 +95898,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - sqlite3TableAffinityStr(v, pTab); + if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0); sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); if( pParse->nested ){ pik_flags = 0; @@ -94510,7 +95974,7 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); - if( pIdx->autoIndex==2 && !HasRowid(pTab) && piDataCur ){ + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) && piDataCur ){ *piDataCur = iIdxCur; } if( aToOpen==0 || aToOpen[i+1] ){ @@ -94726,15 +96190,24 @@ static int xferOptimization( return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ } for(i=0; inCol; i++){ - if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; + if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ return 0; /* Collating sequence must be the same on all columns */ } - if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ + if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } + /* Default values for second and subsequent columns need to match. */ + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) + ){ + return 0; /* Default values must be the same for all columns */ + } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ if( pDestIdx->onError!=OE_None ){ @@ -94803,16 +96276,17 @@ static int xferOptimization( ** ** (3) onError is something other than OE_Abort and OE_Rollback. */ - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); emptyDestTest = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); - emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); sqlite3RowidConstraint(pParse, onError, pDest); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); @@ -94826,7 +96300,7 @@ static int xferOptimization( sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); sqlite3VdbeChangeP4(v, -1, pDest->zName, 0); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); }else{ @@ -94845,15 +96319,15 @@ static int xferOptimization( sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); - addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); } - sqlite3VdbeJumpHere(v, emptySrcTest); + if( emptySrcTest ) sqlite3VdbeJumpHere(v, emptySrcTest); sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ @@ -97087,6 +98561,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** size of historical compatibility. */ case PragTyp_DEFAULT_CACHE_SIZE: { + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ @@ -97104,7 +98579,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cache_size", SQLITE_STATIC); pParse->nMem += 2; - addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); + addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize,iLn); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); @@ -97310,7 +98785,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &iLimit); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -97349,6 +98824,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** file. Before writing to meta[6], check that meta[3] indicates ** that this really is an auto-vacuum capable database. */ + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList setMeta6[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, @@ -97358,7 +98834,7 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_SetCookie, 0, BTREE_INCR_VACUUM, 1}, /* 5 */ }; int iAddr; - iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6); + iAddr = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); sqlite3VdbeChangeP1(v, iAddr, iDb); sqlite3VdbeChangeP1(v, iAddr+1, iDb); sqlite3VdbeChangeP2(v, iAddr+2, iAddr+4); @@ -97384,10 +98860,10 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); - addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_ResultRow, 1); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr); break; } @@ -97437,7 +98913,7 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &sz, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &sz); if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ @@ -97749,13 +99225,15 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -97958,7 +99436,7 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pParse->nErr>0 || pFK==0 ); if( pFK ) break; if( pParse->nTabnTab = i; - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); pIdx = 0; @@ -97974,26 +99452,26 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iKey!=pTab->iPKey ){ sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); sqlite3ColumnDefault(v, pTab, iKey, regRow); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); - sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, - sqlite3VdbeCurrentAddr(v)+3); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, + sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } - sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); + sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ for(j=0; jnCol; j++){ sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); } if( pParent ){ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regRow, pFK->nCol, regKey); - sqlite3VdbeChangeP4(v, -1, - sqlite3IndexAffinityStr(v,pIdx), P4_TRANSIENT); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(v,pIdx), pFK->nCol); sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); } } sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); @@ -98004,7 +99482,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); } - sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrTop); } } @@ -98051,6 +99529,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** messages have been generated, output OK. Otherwise output the ** error message */ + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ { OP_IfNeg, 1, 0, 0}, /* 1 */ @@ -98099,6 +99578,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3CodeVerifySchema(pParse, i); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); @@ -98130,11 +99610,11 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Do the b-tree integrity checks */ sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); sqlite3VdbeChangeP5(v, (u8)i); - addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_Move, 2, 4); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); @@ -98152,6 +99632,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pTab->pIndex==0 ) continue; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); @@ -98162,7 +99643,7 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } pParse->nMem = MAX(pParse->nMem, 8+j); - sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4; @@ -98172,7 +99653,7 @@ SQLITE_PRIVATE void sqlite3Pragma( pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, 0, r1, - pIdx->nColumn); + pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); @@ -98182,13 +99663,13 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); + jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); sqlite3VdbeAddOp0(v, OP_Halt); sqlite3VdbeJumpHere(v, jmp4); sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeResolveLabel(v, jmp3); + sqlite3ResolvePartIdxLabel(pParse, jmp3); } - sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); + sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, @@ -98196,10 +99677,11 @@ SQLITE_PRIVATE void sqlite3Pragma( for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ if( pPk==pIdx ) continue; addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); + sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); @@ -98208,7 +99690,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* SQLITE_OMIT_BTREECOUNT */ } } - addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode); + addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); sqlite3VdbeChangeP2(v, addr, -mxErr); sqlite3VdbeJumpHere(v, addr+1); sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); @@ -98346,7 +99828,7 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_Integer, 0, 1, 0}, /* 1 */ { OP_SetCookie, 0, 0, 1}, /* 2 */ }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); + int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); sqlite3VdbeChangeP1(v, addr+2, iDb); @@ -98358,7 +99840,7 @@ SQLITE_PRIVATE void sqlite3Pragma( { OP_ReadCookie, 0, 1, 0}, /* 1 */ { OP_ResultRow, 1, 1, 0} }; - int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie); + int addr = sqlite3VdbeAddOpList(v, ArraySize(readCookie), readCookie, 0); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP3(v, addr+1, iCookie); @@ -98474,7 +99956,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SOFT_HEAP_LIMIT: { sqlite3_int64 N; - if( zRight && sqlite3Atoi64(zRight, &N, 1000000, SQLITE_UTF8)==SQLITE_OK ){ + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); @@ -99470,6 +100952,34 @@ SQLITE_API int sqlite3_prepare16_v2( ** to handle SELECT statements in SQLite. */ +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure but do not deallocate @@ -99543,7 +101053,6 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( assert( pOffset==0 || pLimit!=0 ); pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; if( db->mallocFailed ) { clearSelect(db, pNew); if( pNew!=&standin ) sqlite3DbFree(db, pNew); @@ -99565,6 +101074,14 @@ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ } } +/* +** Return a pointer to the right-most SELECT statement in a compound. +*/ +static Select *findRightmost(Select *p){ + while( p->pNext ) p = p->pNext; + return p; +} + /* ** Given 1 to 3 identifiers preceding the JOIN keyword, determine the ** type of join. Return an integer constant that expresses that type @@ -99867,34 +101384,73 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } +/* Forward reference */ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +); + /* -** Insert code into "v" that will push the record on the top of the -** stack into the sorter. +** Insert code into "v" that will push the record in register regData +** into the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ - ExprList *pOrderBy, /* The ORDER BY clause */ + SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData /* Register holding data to be sorted */ ){ Vdbe *v = pParse->pVdbe; - int nExpr = pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); + int nExpr = pSort->pOrderBy->nExpr; + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; + int nOBSat = pSort->nOBSat; int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - if( pSelect->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + 1; + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; + pOp->p2 = nKey + 1; + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } - sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( pSelect->iLimit ){ int addr1, addr2; int iLimit; @@ -99903,12 +101459,12 @@ static void pushOntoSorter( }else{ iLimit = pSelect->iLimit; } - addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); + addr1 = sqlite3VdbeAddOp1(v, OP_IfZero, iLimit); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr2); } } @@ -99921,10 +101477,10 @@ static void codeOffset( int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( iOffset>0 && iContinue!=0 ){ + if( iOffset>0 ){ int addr; sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); @@ -99952,7 +101508,7 @@ static void codeDistinct( v = pParse->pVdbe; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); sqlite3ReleaseTempReg(pParse, r1); @@ -99982,19 +101538,6 @@ static int checkForMultiColumnSelectError( } #endif -/* -** An instance of the following object is used to record information about -** how to process the DISTINCT keyword, to simplify passing that information -** into the selectInnerLoop() routine. -*/ -typedef struct DistinctCtx DistinctCtx; -struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ - u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ - int tabTnct; /* Ephemeral table used for DISTINCT processing */ - int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ -}; - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -100009,7 +101552,7 @@ static void selectInnerLoop( Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - ExprList *pOrderBy, /* If not NULL, sort results using this key */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ @@ -100026,20 +101569,28 @@ static void selectInnerLoop( assert( v ); assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; - if( pOrderBy==0 && !hasDistinct ){ + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. */ nResultCol = pEList->nExpr; + if( pDest->iSdst==0 ){ pDest->iSdst = pParse->nMem+1; - pDest->nSdst = nResultCol; pParse->nMem += nResultCol; - }else{ - assert( pDest->nSdst==nResultCol ); + }else if( pDest->iSdst+nResultCol > pParse->nMem ){ + /* This is an error condition that can result, for example, when a SELECT + ** on the right-hand side of an INSERT contains more result columns than + ** there are columns in the table on the left. The error will be caught + ** and reported later. But we need to make sure enough memory is allocated + ** to avoid other spurious errors in the meantime. */ + pParse->nMem += nResultCol; } + pDest->nSdst = nResultCol; regResult = pDest->iSdst; if( srcTab>=0 ){ for(i=0; ia[i].pExpr); if( iiOffset, iContinue); } } @@ -100139,7 +101692,8 @@ static void selectInnerLoop( /* Store the result as data using a unique key. */ - case SRT_DistTable: + case SRT_Fifo: + case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); @@ -100147,20 +101701,20 @@ static void selectInnerLoop( testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE - if( eDest==SRT_DistTable ){ - /* If the destination is DistTable, then cursor (iParm+1) is open + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open ** on an ephemeral index. If the current row is already present ** in the index, do not write it to the output. If not, add the ** current row to the index and proceed with writing it to the ** output table as well. */ int addr = sqlite3VdbeCurrentAddr(v) + 4; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); - assert( pOrderBy==0 ); + assert( pSort==0 ); } #endif - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, r1); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, r1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -100181,12 +101735,12 @@ static void selectInnerLoop( assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); - if( pOrderBy ){ + if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -100211,8 +101765,8 @@ static void selectInnerLoop( */ case SRT_Mem: { assert( nResultCol==1 ); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, regResult); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -100221,18 +101775,14 @@ static void selectInnerLoop( } #endif /* #ifndef SQLITE_OMIT_SUBQUERY */ - /* Send the data to the callback function or to a subroutine. In the - ** case of a subroutine, the subroutine itself is responsible for - ** popping the data from the stack. - */ - case SRT_Coroutine: - case SRT_Output: { + case SRT_Coroutine: /* Send data to a co-routine */ + case SRT_Output: { /* Return the results */ testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); - if( pOrderBy ){ + if( pSort ){ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pOrderBy, p, r1); + pushOntoSorter(pParse, pSort, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); @@ -100262,13 +101812,16 @@ static void selectInnerLoop( r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempRange(pParse, nKey+2); r3 = r2+nKey+1; - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); if( eDest==SRT_DistQueue ){ /* If the destination is DistQueue, then cursor (iParm+1) is open ** on a second ephemeral index that holds all values every previously - ** added to the queue. Only add this new value if it has never before - ** been added */ - addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); } @@ -100306,8 +101859,8 @@ static void selectInnerLoop( ** there is a sorter, in which case the sorter has already limited ** the output for us. */ - if( pOrderBy==0 && p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); + if( pSort==0 && p->iLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } } @@ -100377,7 +101930,12 @@ SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; @@ -100385,15 +101943,15 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); - for(i=0, pItem=pList->a; ia+iStart; ipExpr); if( !pColl ) pColl = db->pDfltColl; - pInfo->aColl[i] = pColl; - pInfo->aSortOrder[i] = pItem->sortOrder; + pInfo->aColl[i-iStart] = pColl; + pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; @@ -100495,24 +102053,31 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ + SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; + int addrOnce = 0; int iTab; int pseudoTab = 0; - ExprList *pOrderBy = p->pOrderBy; - + ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; - int regRow; int regRowid; + int nKey; - iTab = pOrderBy->iECursor; + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ pseudoTab = pParse->nTab++; @@ -100521,19 +102086,23 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - if( p->selFlags & SF_UseSorter ){ + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ - addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); } switch( eDest ){ case SRT_Table: @@ -100588,15 +102157,13 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - if( p->selFlags & SF_UseSorter ){ - sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); }else{ - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); - } } /* @@ -100962,7 +102529,7 @@ static int selectColumnsFromExprList( char *zNewName; int k; for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){} - if( zName[k]==':' ) nName = k; + if( k>=0 && zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); @@ -101056,7 +102623,7 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -101075,11 +102642,13 @@ SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ Vdbe *v = pParse->pVdbe; if( v==0 ){ v = pParse->pVdbe = sqlite3VdbeCreate(pParse); -#ifndef SQLITE_OMIT_TRACE - if( v ){ - sqlite3VdbeAddOp0(v, OP_Trace); + if( v ) sqlite3VdbeAddOp0(v, OP_Init); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; } -#endif + } return v; } @@ -101137,22 +102706,22 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); - sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); + sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak); VdbeCoverage(v); } if( p->pOffset ){ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ sqlite3ExprCode(pParse, p->pOffset, iOffset); - sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); + addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); sqlite3VdbeJumpHere(v, addr1); } @@ -101272,7 +102841,7 @@ static void generateWithRecursiveQuery( int regCurrent; /* Register holding Current table */ int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; /* How to write to Queue */ + int eDest = SRT_Fifo; /* How to write to Queue */ SelectDest destQueue; /* SelectDest targetting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ @@ -101304,13 +102873,13 @@ static void generateWithRecursiveQuery( /* Allocate cursors numbers for Queue and Distinct. The cursor number for ** the Distinct table must be exactly one greater than Queue in order - ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ iQueue = pParse->nTab++; if( p->op==TK_UNION ){ - eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; iDistinct = pParse->nTab++; }else{ - eDest = pOrderBy ? SRT_Queue : SRT_Table; + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; } sqlite3SelectDestInit(&destQueue, eDest, iQueue); @@ -101335,11 +102904,13 @@ static void generateWithRecursiveQuery( p->pOrderBy = 0; /* Store the results of the setup-query in Queue. */ + pSetup->pNext = 0; rc = sqlite3Select(pParse, pSetup, &destQueue); + pSetup->pNext = p; if( rc ) goto end_of_recursive_query; /* Find the next row in the Queue and output that row */ - addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v); /* Transfer the next row in Queue over to Current */ sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ @@ -101355,7 +102926,10 @@ static void generateWithRecursiveQuery( codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, 0, 0, pDest, addrCont, addrBreak); - if( regLimit ) sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + if( regLimit ){ + sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + VdbeCoverage(v); + } sqlite3VdbeResolveLabel(v, addrCont); /* Execute the recursive SELECT taking the single row in Current as @@ -101371,6 +102945,7 @@ static void generateWithRecursiveQuery( sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); p->pOrderBy = pOrderBy; p->pLimit = pLimit; p->pOffset = pOffset; @@ -101440,8 +103015,6 @@ static int multiSelect( assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; - assert( pPrior->pRightmost!=pPrior ); - assert( pPrior->pRightmost==p->pRightmost ); dest = *pDest; if( pPrior->pOrderBy ){ sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", @@ -101517,7 +103090,7 @@ static int multiSelect( p->iLimit = pPrior->iLimit; p->iOffset = pPrior->iOffset; if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); + addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); } explainSetInteger(iSub2, pParse->iNextSelectId); @@ -101549,12 +103122,10 @@ static int multiSelect( testcase( p->op==TK_EXCEPT ); testcase( p->op==TK_UNION ); priorOp = SRT_Union; - if( dest.eDest==priorOp && ALWAYS(!p->pLimit &&!p->pOffset) ){ + if( dest.eDest==priorOp ){ /* We can reuse a temporary table generated by a SELECT to our ** right. */ - assert( p->pRightmost!=p ); /* Can only happen for leftward elements - ** of a 3-way or more compound */ assert( p->pLimit==0 ); /* Not allowed on leftward elements */ assert( p->pOffset==0 ); /* Not allowed on leftward elements */ unionTab = dest.iSDParm; @@ -101567,7 +103138,7 @@ static int multiSelect( addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; + findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); } @@ -101626,12 +103197,12 @@ static int multiSelect( iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); } @@ -101656,7 +103227,7 @@ static int multiSelect( addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; - p->pRightmost->selFlags |= SF_UsesEphemeral; + findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); /* Code the SELECTs to our left into temporary table "tab1". @@ -101701,15 +103272,15 @@ static int multiSelect( iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); @@ -101735,7 +103306,7 @@ static int multiSelect( CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ int nCol; /* Number of columns in result set */ - assert( p->pRightmost==p ); + assert( p->pNext==0 ); nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ @@ -101816,10 +103387,10 @@ static int generateOutputSubroutine( */ if( regPrev ){ int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); + j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); + sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); sqlite3VdbeJumpHere(v, j1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); @@ -101920,7 +103491,7 @@ static int generateOutputSubroutine( /* Jump to the end of the loop if the LIMIT is reached. */ if( p->iLimit ){ - sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); + sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } /* Generate the subroutine return @@ -102028,9 +103599,7 @@ static int multiSelectOrderBy( SelectDest destA; /* Destination for coroutine A */ SelectDest destB; /* Destination for coroutine B */ int regAddrA; /* Address register for select-A coroutine */ - int regEofA; /* Flag to indicate when select-A is complete */ int regAddrB; /* Address register for select-B coroutine */ - int regEofB; /* Flag to indicate when select-B is complete */ int addrSelectA; /* Address of the select-A coroutine */ int addrSelectB; /* Address of the select-B coroutine */ int regOutA; /* Address register for the output-A subroutine */ @@ -102038,6 +103607,7 @@ static int multiSelectOrderBy( int addrOutA; /* Address of the output-A subroutine */ int addrOutB = 0; /* Address of the output-B subroutine */ int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */ int addrEofB; /* Address of the select-B-exhausted subroutine */ int addrAltB; /* Address of the ApPrior = 0; + pPrior->pNext = 0; sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); if( pPrior->pPrior==0 ){ sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); @@ -102174,37 +103745,30 @@ static int multiSelectOrderBy( p->pOffset = 0; regAddrA = ++pParse->nMem; - regEofA = ++pParse->nMem; regAddrB = ++pParse->nMem; - regEofB = ++pParse->nMem; regOutA = ++pParse->nMem; regOutB = ++pParse->nMem; sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); - /* Jump past the various subroutines and coroutines to the main - ** merge loop - */ - j1 = sqlite3VdbeAddOp0(v, OP_Goto); - addrSelectA = sqlite3VdbeCurrentAddr(v); - - /* Generate a coroutine to evaluate the SELECT statement to the ** left of the compound operator - the "A" select. */ - VdbeNoopComment((v, "Begin coroutine for left SELECT")); + addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - VdbeNoopComment((v, "End coroutine for left SELECT")); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); + sqlite3VdbeJumpHere(v, j1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ - addrSelectB = sqlite3VdbeCurrentAddr(v); - VdbeNoopComment((v, "Begin coroutine for right SELECT")); + addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; + j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -102213,9 +103777,7 @@ static int multiSelectOrderBy( sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - VdbeNoopComment((v, "End coroutine for right SELECT")); + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrB); /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. @@ -102239,13 +103801,13 @@ static int multiSelectOrderBy( /* Generate a subroutine to run when the results from select A ** are exhausted and only data in select B remains. */ - VdbeNoopComment((v, "eof-A subroutine")); if( op==TK_EXCEPT || op==TK_INTERSECT ){ - addrEofA = sqlite3VdbeAddOp2(v, OP_Goto, 0, labelEnd); + addrEofA_noB = addrEofA = labelEnd; }else{ - addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); + VdbeNoopComment((v, "eof-A subroutine")); + addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); + VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); p->nSelectRow += pPrior->nSelectRow; } @@ -102258,9 +103820,8 @@ static int multiSelectOrderBy( if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ VdbeNoopComment((v, "eof-B subroutine")); - addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); - sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); + addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofB); } @@ -102268,8 +103829,7 @@ static int multiSelectOrderBy( */ VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); /* Generate code to handle the case of A==B @@ -102282,8 +103842,7 @@ static int multiSelectOrderBy( }else{ VdbeNoopComment((v, "A-eq-B subroutine")); addrAeqB = - sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); } @@ -102294,19 +103853,14 @@ static int multiSelectOrderBy( if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); } - sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, j1); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofA); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEofB); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrA, addrSelectA); - sqlite3VdbeAddOp2(v, OP_Gosub, regAddrB, addrSelectB); - sqlite3VdbeAddOp2(v, OP_If, regEofA, addrEofA); - sqlite3VdbeAddOp2(v, OP_If, regEofB, addrEofB); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); /* Implement the main merge loop */ @@ -102315,7 +103869,7 @@ static int multiSelectOrderBy( sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO); sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); /* Jump to the this point in order to terminate the query. */ @@ -102335,6 +103889,7 @@ static int multiSelectOrderBy( sqlite3SelectDelete(db, p->pPrior); } p->pPrior = pPrior; + pPrior->pNext = p; /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ @@ -102600,7 +104155,7 @@ static int flattenSubquery( ** and (14). */ if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ if( pSub->pOffset ) return 0; /* Restriction (14) */ - if( p->pRightmost && pSub->pLimit ){ + if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ @@ -102751,14 +104306,14 @@ static int flattenSubquery( p->pOrderBy = pOrderBy; p->pSrc = pSrc; p->op = TK_ALL; - p->pRightmost = 0; if( pNew==0 ){ - pNew = pPrior; + p->pPrior = pPrior; }else{ pNew->pPrior = pPrior; - pNew->pRightmost = 0; + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = p; + p->pPrior = pNew; } - p->pPrior = pNew; if( db->mallocFailed ) return 1; } @@ -103097,6 +104652,10 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ pNew->pHaving = 0; pNew->pOrderBy = 0; p->pPrior = 0; + p->pNext = 0; + p->selFlags &= ~SF_Compound; + assert( pNew->pPrior!=0 ); + pNew->pPrior->pNext = pNew; pNew->pLimit = 0; pNew->pOffset = 0; return WRC_Continue; @@ -103203,7 +104762,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -103284,9 +104843,10 @@ static int withExpand( */ static void selectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - if( p->pWith ){ - assert( pParse->pWith==p->pWith ); - pParse->pWith = p->pWith->pOuter; + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; } } #else @@ -103336,7 +104896,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - sqlite3WithPush(pParse, p->pWith, 0); + sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -103378,7 +104938,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -103757,7 +105317,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -103849,7 +105409,7 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ ** values to an OP_Copy. */ if( regHit ){ - addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ @@ -103873,10 +105433,11 @@ static void explainSimpleCount( Index *pIdx /* Index used to optimize scan, or NULL */ ){ if( pParse->explain==2 ){ + int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s", - pTab->zName, - pIdx ? " USING COVERING INDEX " : "", - pIdx ? pIdx->zName : "" + pTab->zName, + bCover ? " USING COVERING INDEX " : "", + bCover ? pIdx->zName : "" ); sqlite3VdbeAddOp4( pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC @@ -103912,12 +105473,11 @@ SQLITE_PRIVATE int sqlite3Select( ExprList *pEList; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ - ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ int rc = 1; /* Value to return from this function */ - int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -103934,9 +105494,15 @@ SQLITE_PRIVATE int sqlite3Select( if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); @@ -103944,7 +105510,8 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - pOrderBy = p->pOrderBy; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ @@ -104008,42 +105575,24 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags |= SF_Aggregate; } i = -1; - }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) + }else if( pTabList->nSrc==1 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ - int addrTop; - int addrEof; + int addrTop = sqlite3VdbeCurrentAddr(v)+1; pItem->regReturn = ++pParse->nMem; - addrEof = ++pParse->nMem; - /* Before coding the OP_Goto to jump to the start of the main routine, - ** ensure that the jump to the verify-schema routine has already - ** been coded. Otherwise, the verify-schema would likely be coded as - ** part of the co-routine. If the main routine then accessed the - ** database before invoking the co-routine for the first time (for - ** example to initialize a LIMIT register from a sub-select), it would - ** be doing so without having verified the schema version and obtained - ** the required db locks. See ticket d6b36be38. */ - sqlite3CodeVerifySchema(pParse, -1); - sqlite3VdbeAddOp0(v, OP_Goto); - addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor); - sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, "coroutine for %s", pItem->pTab->zName)); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%s", pItem->pTab->zName)); pItem->addrFillSub = addrTop; - sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof); - sqlite3VdbeChangeP5(v, 1); sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; - sqlite3VdbeChangeP2(v, addrTop, dest.iSdst); - sqlite3VdbeChangeP3(v, addrTop, dest.nSdst); - sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof); - sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn); - VdbeComment((v, "end %s", pItem->pTab->zName)); + pItem->regResult = dest.iSdst; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else{ @@ -104059,17 +105608,19 @@ SQLITE_PRIVATE int sqlite3Select( pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; - VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3CodeOnce(pParse); + onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + }else{ + VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); @@ -104082,7 +105633,7 @@ SQLITE_PRIVATE int sqlite3Select( pParse->nHeight -= sqlite3SelectExprHeight(p); pTabList = p->pSrc; if( !IgnorableOrderby(pDest) ){ - pOrderBy = p->pOrderBy; + sSort.pOrderBy = p->pOrderBy; } } pEList = p->pEList; @@ -104096,39 +105647,12 @@ SQLITE_PRIVATE int sqlite3Select( /* If there is are a sequence of queries, do the earlier ones first. */ if( p->pPrior ){ - if( p->pRightmost==0 ){ - Select *pLoop, *pRight = 0; - int cnt = 0; - int mxSelect; - for(pLoop=p; pLoop; pLoop=pLoop->pPrior, cnt++){ - pLoop->pRightmost = p; - pLoop->pNext = pRight; - pRight = pLoop; - } - mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; - if( mxSelect && cnt>mxSelect ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - goto select_end; - } - } rc = multiSelect(pParse, p, pDest); explainSetInteger(pParse->iSelectId, iRestoreSelectId); return rc; } #endif - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then disable the ORDER BY clause since the GROUP BY - ** will cause elements to come out in the correct order. This is - ** an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER - ** to disable this optimization for testing purposes. - */ - if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy, -1)==0 - && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ - pOrderBy = 0; - } - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: @@ -104145,12 +105669,12 @@ SQLITE_PRIVATE int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - pOrderBy = 0; + sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ @@ -104164,16 +105688,16 @@ SQLITE_PRIVATE int sqlite3Select( ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); - pOrderBy->iECursor = pParse->nTab++; - p->addrOpenEphm[2] = addrSortIndex = + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+2, 0, + sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); }else{ - addrSortIndex = -1; + sSort.addrSortIndex = -1; } /* If the output is destined for a temporary table, open that table. @@ -104187,9 +105711,9 @@ SQLITE_PRIVATE int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); - if( p->iLimit==0 && addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; - p->selFlags |= SF_UseSorter; + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sSort.sortFlags |= SORTFLAG_UseSorter; } /* Open a virtual index to use for the distinct set. @@ -104198,7 +105722,7 @@ SQLITE_PRIVATE int sqlite3Select( sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList, 0), + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; @@ -104211,8 +105735,8 @@ SQLITE_PRIVATE int sqlite3Select( u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, - wctrlFlags, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -104220,19 +105744,23 @@ SQLITE_PRIVATE int sqlite3Select( if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } - if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0; + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ - if( addrSortIndex>=0 && pOrderBy==0 ){ - sqlite3VdbeChangeToNoop(v, addrSortIndex); - p->addrOpenEphm[2] = -1; + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -104253,6 +105781,7 @@ SQLITE_PRIVATE int sqlite3Select( int addrEnd; /* End of processing for this SELECT */ int sortPTab = 0; /* Pseudotable used to decode sorting results */ int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -104272,6 +105801,18 @@ SQLITE_PRIVATE int sqlite3Select( p->nSelectRow = 1; } + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY may use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(v); @@ -104288,7 +105829,7 @@ SQLITE_PRIVATE int sqlite3Select( sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); - sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } @@ -104322,7 +105863,7 @@ SQLITE_PRIVATE int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -104351,10 +105892,11 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, - WHERE_GROUPBY, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + ); if( pWInfo==0 ) goto select_end; - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo @@ -104414,9 +105956,24 @@ SQLITE_PRIVATE int sqlite3Select( sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); - VdbeComment((v, "GROUP BY sort")); + VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); + + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Evaluate the current GROUP BY terms and store in b0, b1, b2... @@ -104441,7 +105998,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); + sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code @@ -104455,7 +106012,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); VdbeComment((v, "output one row")); - sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); VdbeComment((v, "reset accumulator")); @@ -104472,6 +106029,7 @@ SQLITE_PRIVATE int sqlite3Select( */ if( groupBySort ){ sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); + VdbeCoverage(v); }else{ sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); @@ -104499,12 +106057,12 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); sqlite3VdbeResolveLabel(v, addrOutputRow); addrOutputRow = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, &sSort, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -104636,7 +106194,7 @@ SQLITE_PRIVATE int sqlite3Select( } updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -104645,7 +106203,7 @@ SQLITE_PRIVATE int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); } - pOrderBy = 0; + sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); @@ -104662,9 +106220,9 @@ SQLITE_PRIVATE int sqlite3Select( /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ - if( pOrderBy ){ - explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, v, pEList->nExpr, pDest); + if( sSort.pOrderBy ){ + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } /* Jump here to skip this query @@ -104772,10 +106330,6 @@ SQLITE_PRIVATE void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ sqlite3ExplainPrintf(pVdbe, "(null-select)"); return; } - while( p->pPrior ){ - p->pPrior->pNext = p; - p = p->pPrior; - } sqlite3ExplainPush(pVdbe); while( p ){ explainOneSelect(pVdbe, p); @@ -105560,6 +107114,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ int base; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList dropTrigger[] = { { OP_Rewind, 0, ADDR(9), 0}, { OP_String8, 0, 1, 0}, /* 1 */ @@ -105574,7 +107129,7 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3OpenMasterTable(pParse, iDb); - base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger); + base = sqlite3VdbeAddOpList(v, ArraySize(dropTrigger), dropTrigger, iLn); sqlite3VdbeChangeP4(v, base+1, pTrigger->zName, P4_TRANSIENT); sqlite3VdbeChangeP4(v, base+4, "trigger", P4_STATIC); sqlite3ChangeCookie(pParse, iDb); @@ -105720,15 +107275,7 @@ static int codeTriggerProgram( ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy */ pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; - - /* Clear the cookieGoto flag. When coding triggers, the cookieGoto - ** variable is used as a flag to indicate to sqlite3ExprCodeConstants() - ** that it is not safe to refactor constants (this happens after the - ** start of the first loop in the SQL statement is coded - at that - ** point code may be conditionally executed, so it is no longer safe to - ** initialize constant register values). */ - assert( pParse->cookieGoto==0 || pParse->cookieGoto==-1 ); - pParse->cookieGoto = 0; + assert( pParse->okConstFactor==0 ); switch( pStep->op ){ case TK_UPDATE: { @@ -106314,7 +107861,7 @@ SQLITE_PRIVATE void sqlite3Update( iIdxCur = iDataCur+1; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ - if( pIdx->autoIndex==2 && pPk!=0 ){ + if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){ iDataCur = pParse->nTab; pTabList->a[0].iCursor = iDataCur; } @@ -106517,7 +108064,7 @@ SQLITE_PRIVATE void sqlite3Update( regKey = iPk; }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(v, pPk), P4_TRANSIENT); + sqlite3IndexAffinityStr(v, pPk), nPk); sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); } sqlite3WhereEnd(pWInfo); @@ -106561,18 +108108,23 @@ SQLITE_PRIVATE void sqlite3Update( if( aToOpen[iDataCur-iBaseCur] ){ assert( pPk!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); + VdbeCoverageNeverTaken(v); } labelContinue = labelBreak; sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverage(v); }else if( pPk ){ labelContinue = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); addrTop = sqlite3VdbeAddOp2(v, OP_RowKey, iEph, regKey); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0); + VdbeCoverage(v); }else{ labelContinue = sqlite3VdbeAddOp3(v, OP_RowSetRead, regRowSet, labelBreak, regOldRowid); + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); } /* If the record number will change, set register regNewRowid to @@ -106582,7 +108134,7 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngKey || pTrigger || hasFK || regOldRowid==regNewRowid ); if( chngRowid ){ sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); - sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); } /* Compute the old pre-UPDATE content of the row being changed, if that @@ -106651,8 +108203,7 @@ SQLITE_PRIVATE void sqlite3Update( ** verified. One could argue that this is wrong. */ if( tmask&TRIGGER_BEFORE ){ - sqlite3VdbeAddOp2(v, OP_Affinity, regNew, pTab->nCol); - sqlite3TableAffinityStr(v, pTab); + sqlite3TableAffinity(v, pTab, regNew); sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue); @@ -106664,8 +108215,10 @@ SQLITE_PRIVATE void sqlite3Update( */ if( pPk ){ sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue,regKey,nKey); + VdbeCoverage(v); }else{ sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); + VdbeCoverage(v); } /* If it did not delete it, the row-trigger may still have modified @@ -106701,6 +108254,7 @@ SQLITE_PRIVATE void sqlite3Update( }else{ j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } + VdbeCoverageNeverTaken(v); } sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); @@ -106744,7 +108298,7 @@ SQLITE_PRIVATE void sqlite3Update( /* Nothing to do at end-of-loop for a single-pass */ }else if( pPk ){ sqlite3VdbeResolveLabel(v, labelContinue); - sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); + sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Goto, 0, labelContinue); } @@ -106873,7 +108427,7 @@ static void updateVirtualTable( /* Generate code to scan the ephemeral table and call VUpdate. */ iReg = ++pParse->nMem; pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); + addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); for(i=0; inCol; i++){ @@ -106883,7 +108437,7 @@ static void updateVirtualTable( sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr); sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); @@ -108455,7 +110009,7 @@ struct WhereLevel { int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ - u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { @@ -108506,7 +110060,7 @@ struct WhereLoop { struct { /* Information for virtual tables */ int idxNum; /* Index number */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */ - u8 isOrdered; /* True if satisfies ORDER BY */ + i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ } vtab; @@ -108568,8 +110122,7 @@ struct WherePath { Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ - u8 isOrdered; /* True if this path satisfies ORDER BY */ - u8 isOrderedValid; /* True if the isOrdered field is valid */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; @@ -108783,7 +110336,8 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 bOBSat; /* ORDER BY satisfied by indices */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 sorted; /* True if really sorted (not just grouped) */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ @@ -108842,6 +110396,7 @@ struct WhereInfo { #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ +#define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ /************** End of whereInt.h ********************************************/ /************** Continuing where we left off in where.c **********************/ @@ -108866,7 +110421,7 @@ SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** Return FALSE if the output needs to be sorted. */ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->bOBSat!=0; + return pWInfo->nOBSat; } /* @@ -108874,6 +110429,7 @@ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ ** immediately with the next row of a WHERE clause. */ SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); return pWInfo->iContinue; } @@ -109053,7 +110609,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -109370,7 +110926,7 @@ static WhereTerm *whereScanInit( if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nKeyCol) ) return 0; + if( NEVER(j>pIdx->nColumn) ) return 0; } pScan->zCollName = pIdx->azColl[j]; }else{ @@ -110320,8 +111876,7 @@ static int isDistinctRedundant( ** Estimate the logarithm of the input value to base 2. */ static LogEst estLog(LogEst N){ - LogEst x = sqlite3LogEst(N); - return x>33 ? x - 33 : 0; + return N<=10 ? 0 : sqlite3LogEst(N) - 33; } /* @@ -110428,7 +111983,7 @@ static void constructAutomaticIndex( ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - addrInit = sqlite3CodeOnce(pParse); + addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ @@ -110535,12 +112090,12 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); - sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); @@ -110740,7 +112295,7 @@ static void whereKeyStats( assert( pRec->nField>0 && iColnSampleCol ); do{ iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); if( res<0 ){ iMin = iTest+1; }else{ @@ -110755,16 +112310,16 @@ static void whereKeyStats( if( res==0 ){ /* If (res==0) is true, then sample $i must be equal to pRec */ assert( inSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) || pParse->db->mallocFailed ); }else{ /* Otherwise, pRec must be smaller than sample $i and larger than ** sample ($i-1). */ assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 || pParse->db->mallocFailed ); assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 || pParse->db->mallocFailed ); } #endif /* ifdef SQLITE_DEBUG */ @@ -110782,10 +112337,11 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } - aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); + aStat[1] = pIdx->aAvgEq[iCol]; if( iLower>=iUpper ){ iGap = 0; }else{ @@ -110801,6 +112357,138 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -110837,9 +112525,9 @@ static void whereKeyStats( ** to account for the range contraints pLower and pUpper. ** ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be -** used, each range inequality reduces the search space by a factor of 4. -** Hence a pair of constraints (x>? AND x? AND xu.btree.nEq; if( p->nSample>0 - && nEq==pBuilder->nRecValid && nEqnSampleCol && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - u8 aff; - - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - */ - tRowcnt iLower; - tRowcnt iUpper; + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + */ + tRowcnt iLower; + tRowcnt iUpper; - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = p->aiRowEst[0]; - }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; - } - - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; + }else{ + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; + } + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; + iLower = a[0]; + iUpper = a[0] + a[1]; } - } - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); - if( iNewpExpr->pRight; + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + } } - } - pBuilder->pRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); - }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewnOut = (LogEst)nOut; + WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", + (u32)iLower, (u32)iUpper, nOut)); + return SQLITE_OK; } - pLoop->nOut = (LogEst)nOut; - WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", - (u32)iLower, (u32)iUpper, nOut)); - return SQLITE_OK; + }else{ + int bDone = 0; + rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); + if( bDone ) return rc; } } #else @@ -110953,17 +112646,18 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; + + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -111003,7 +112697,7 @@ static int whereEqualScanEst( int bOk; assert( nEq>=1 ); - assert( nEq<=(p->nKeyCol+1) ); + assert( nEq<=p->nColumn ); assert( p->aSample!=0 ); assert( p->nSample>0 ); assert( pBuilder->nRecValidp->nKeyCol ){ + if( nEq>=p->nColumn ){ *pnRow = 1; return SQLITE_OK; } @@ -111060,6 +112754,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -111068,14 +112763,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -111216,6 +112911,8 @@ static int codeEqualityTerm( } iTab = pX->iTable; sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); pLoop->wsFlags |= WHERE_IN_ABLE; if( pLevel->u.in.nIn==0 ){ @@ -111235,7 +112932,7 @@ static int codeEqualityTerm( pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); + sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); }else{ pLevel->u.in.nIn = 0; } @@ -111330,10 +113027,14 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); - pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLt:OP_SeekGt), + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); sqlite3VdbeJumpHere(v, j); for(j=0; jeOperator & WO_IN ); if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ Expr *pRight = pTerm->pExpr->pRight; - sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk); + if( sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } if( zAff ){ if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){ zAff[j] = SQLITE_AFF_NONE; @@ -111437,7 +113141,7 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ txt.db = db; sqlite3StrAccumAppend(&txt, " (", 2); for(i=0; inKeyCol ) ? "rowid" : aCol[aiColumn[i]].zName; + char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName; if( i>=nSkip ){ explainAppendTerm(&txt, i, z, "="); }else{ @@ -111450,11 +113154,11 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i, z, "<"); } sqlite3StrAccumAppend(&txt, ")", 1); @@ -111509,13 +113213,20 @@ static void explainOneScan( if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 && ALWAYS(pLoop->u.btree.pIndex!=0) ){ + const char *zFmt; + Index *pIdx = pLoop->u.btree.pIndex; char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); - zMsg = sqlite3MAppendf(db, zMsg, - ((flags & WHERE_AUTO_INDEX) ? - "%s USING AUTOMATIC %sINDEX%.0s%s" : - "%s USING %sINDEX %s%s"), - zMsg, ((flags & WHERE_IDX_ONLY) ? "COVERING " : ""), - pLoop->u.btree.pIndex->zName, zWhere); + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "%s USING COVERING INDEX %s%s"; + }else{ + zFmt = "%s USING INDEX %s%s"; + } + zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere); sqlite3DbFree(db, zWhere); }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); @@ -111612,10 +113323,10 @@ static Bitmask codeOneLoopStart( /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->viaCoroutine ){ int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield); - pLevel->p2 = sqlite3VdbeAddOp1(v, OP_Yield, regYield); - VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName)); - sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); pLevel->op = OP_Goto; }else @@ -111647,6 +113358,7 @@ static Bitmask codeOneLoopStart( sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pLoop->u.vtab.idxStr, pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + VdbeCoverage(v); pLoop->u.vtab.needFree = 0; for(j=0; ju.vtab.omitMask>>j)&1 ){ @@ -111657,7 +113369,7 @@ static Bitmask codeOneLoopStart( pLevel->p1 = iCur; pLevel->p2 = sqlite3VdbeCurrentAddr(v); sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -111670,16 +113382,18 @@ static Bitmask codeOneLoopStart( ** construct. */ assert( pLoop->u.btree.nEq==1 ); - iReleaseReg = sqlite3GetTempReg(pParse); pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); VdbeComment((v, "pk")); @@ -111713,10 +113427,10 @@ static Bitmask codeOneLoopStart( ** seek opcodes. It depends on a particular ordering of TK_xx */ const u8 aMoveOp[] = { - /* TK_GT */ OP_SeekGt, - /* TK_LE */ OP_SeekLe, - /* TK_LT */ OP_SeekLt, - /* TK_GE */ OP_SeekGe + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ @@ -111730,11 +113444,17 @@ static Bitmask codeOneLoopStart( r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); disableTerm(pLevel, pStart); }else{ sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); } if( pEnd ){ Expr *pX; @@ -111758,10 +113478,14 @@ static Bitmask codeOneLoopStart( pLevel->p2 = start; assert( pLevel->p5==0 ); if( testOp!=OP_Noop ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } }else if( pLoop->wsFlags & WHERE_INDEXED ){ @@ -111801,20 +113525,19 @@ static Bitmask codeOneLoopStart( 0, OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ OP_Last, /* 3: (!start_constraints && startEq && bRev) */ - OP_SeekGt, /* 4: (start_constraints && !startEq && !bRev) */ - OP_SeekLt, /* 5: (start_constraints && !startEq && bRev) */ - OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ - OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ }; static const u8 aEndOp[] = { - OP_Noop, /* 0: (!end_constraints) */ - OP_IdxGE, /* 1: (end_constraints && !bRev) */ - OP_IdxLT /* 2: (end_constraints && bRev) */ + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ }; u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ - int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ int startEq; /* True if range start uses ==, >= or <= */ @@ -111827,6 +113550,8 @@ static Bitmask codeOneLoopStart( int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ char cEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; @@ -111840,12 +113565,15 @@ static Bitmask codeOneLoopStart( ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pWInfo->bOBSat!=0) + && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ assert( pLoop->u.btree.nSkip==0 ); - isMinQuery = 1; + bSeekPastNull = 1; nExtraReg = 1; } @@ -111860,7 +113588,14 @@ static Bitmask codeOneLoopStart( if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; + if( pRangeStart==0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ + bSeekPastNull = 1; + } } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers @@ -111879,6 +113614,7 @@ static Bitmask codeOneLoopStart( || (bRev && pIdx->nKeyCol==nEq) ){ SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); } testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); @@ -111894,8 +113630,11 @@ static Bitmask codeOneLoopStart( if( pRangeStart ){ Expr *pRight = pRangeStart->pExpr->pRight; sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeStart->wtFlags & TERM_VNULL)==0 ){ - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } if( zStartAff ){ if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){ @@ -111910,22 +113649,23 @@ static Bitmask codeOneLoopStart( } nConstraint++; testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); - }else if( isMinQuery ){ + }else if( bSeekPastNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); nConstraint++; startEq = 0; start_constraints = 1; } - codeApplyAffinity(pParse, regBase, nConstraint, zStartAff); + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); - testcase( op==OP_Rewind ); - testcase( op==OP_Last ); - testcase( op==OP_SeekGt ); - testcase( op==OP_SeekGe ); - testcase( op==OP_SeekLe ); - testcase( op==OP_SeekLt ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); /* Load the value for the inequality constraint at the end of the ** range (if any). @@ -111935,8 +113675,11 @@ static Bitmask codeOneLoopStart( Expr *pRight = pRangeEnd->pExpr->pRight; sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); sqlite3ExprCode(pParse, pRight, regBase+nEq); - if( (pRangeEnd->wtFlags & TERM_VNULL)==0 ){ - sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); } if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_NONE && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) @@ -111945,6 +113688,10 @@ static Bitmask codeOneLoopStart( } nConstraint++; testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + }else if( bStopAtNull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + nConstraint++; } sqlite3DbFree(db, zStartAff); @@ -111952,32 +113699,14 @@ static Bitmask codeOneLoopStart( pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ - op = aEndOp[(pRangeEnd || nEq) * (1 + bRev)]; - testcase( op==OP_Noop ); - testcase( op==OP_IdxGE ); - testcase( op==OP_IdxLT ); - if( op!=OP_Noop ){ + if( nConstraint ){ + op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0); - } - - /* If there are inequality constraints, check that the value - ** of the table column that the inequality contrains is not NULL. - ** If it is, jump to the next iteration of the loop. - */ - r1 = sqlite3GetTempReg(pParse); - testcase( pLoop->wsFlags & WHERE_BTM_LIMIT ); - testcase( pLoop->wsFlags & WHERE_TOP_LIMIT ); - if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - && (j = pIdx->aiColumn[nEq])>=0 - && pIdx->pTable->aCol[j].notNull==0 - && (nEq || (pLoop->wsFlags & WHERE_BTM_LIMIT)==0) - ){ - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); - VdbeComment((v, "%s", pIdx->pTable->aCol[j].zName)); - sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } - sqlite3ReleaseTempReg(pParse, r1); /* Seek the table cursor, if required */ disableTerm(pLevel, pRangeStart); @@ -111985,11 +113714,11 @@ static Bitmask codeOneLoopStart( if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ - }else{ + }else if( iCur!=iIdxCur ){ Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ @@ -111997,7 +113726,7 @@ static Bitmask codeOneLoopStart( sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); } sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, - iRowidReg, pPk->nKeyCol); + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } /* Record the instruction used to terminate the loop. Disable @@ -112011,6 +113740,7 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ @@ -112058,6 +113788,10 @@ static Bitmask codeOneLoopStart( ** ** B: ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephermeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** */ WhereClause *pOrWc; /* The OR-clause broken out into subterms */ SrcList *pOrTab; /* Shortened table list or OR-clause generation */ @@ -112071,7 +113805,9 @@ static Bitmask codeOneLoopStart( int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); @@ -112104,7 +113840,8 @@ static Bitmask codeOneLoopStart( } /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. + ** equivalent to an empty rowset. Or, create an ephermeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. ** ** Also initialize regReturn to contain the address of the instruction ** immediately following the OP_Return at the bottom of the loop. This @@ -112115,9 +113852,16 @@ static Bitmask codeOneLoopStart( ** called on an uninitialized cursor. */ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - regRowset = ++pParse->nMem; + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } regRowid = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); } iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); @@ -112153,35 +113897,88 @@ static Bitmask codeOneLoopStart( } } + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ - WhereInfo *pSubWInfo; /* Info for single OR-term scan */ - Expr *pOrExpr = pOrTerm->pExpr; + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + int j1 = 0; /* Address of jump operation */ if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); + wctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; - r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid, 0); - sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, - sqlite3VdbeCurrentAddr(v)+2, r, iSet); + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0); + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } } + + /* Invoke the main loop body as a subroutine */ sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( j1 ) sqlite3VdbeJumpHere(v, j1); + /* The pSubWInfo->untestedTerms flag means that this OR term ** contained one or more AND term from a notReady table. The ** terms from the notReady table could not be tested and will @@ -112205,9 +114002,11 @@ static Bitmask codeOneLoopStart( assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; }else{ pCov = 0; } @@ -112247,6 +114046,8 @@ static Bitmask codeOneLoopStart( pLevel->op = aStep[bRev]; pLevel->p1 = iCur; pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } } @@ -112328,7 +114129,6 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } } - sqlite3ReleaseTempReg(pParse, iReleaseReg); return pLevel->notReady; } @@ -112385,7 +114185,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %-19s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" f %04x N %d", p->wsFlags, p->nLTerm); + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); #ifdef SQLITE_ENABLE_TREE_EXPLAIN /* If the 0x100 bit of wheretracing is set, then show all of the constraint @@ -112507,6 +114307,161 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } +/* +** Return TRUE if both of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. +*/ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ + int i, j; + if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + } + for(i=pX->nLTerm-1; i>=0; i--){ + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + return 1; /* All conditions meet */ +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: +** +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate +** +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. +** +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; + } + } +} + +/* +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. +** +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. +** +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. +** +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. +*/ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + + /* Any loop using an appliation-defined index (or PRIMARY KEY or + ** UNIQUE constraint) with one or more == constraints is better + ** than an automatic index. */ + if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 + && (p->prereq & pTemplate->prereq)==pTemplate->prereq + ){ + break; + } + + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ + ){ + return 0; /* Discard pTemplate */ + } + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ + ){ + assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ + break; /* Cause p to be overwritten by pTemplate */ + } + } + return ppPrev; +} + /* ** Insert or replace a WhereLoop entry using the template supplied. ** @@ -112516,25 +114471,23 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** fewer dependencies than the template. Otherwise a new WhereLoop is ** added based on the template. ** -** If pBuilder->pOrSet is not NULL then we only care about only the +** If pBuilder->pOrSet is not NULL then we care about only the ** prerequisites and rRun and nOut costs of the N best loops. That ** information is gathered in the pBuilder->pOrSet object. This special ** processing mode is used only for OR clause processing. ** ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we ** still might overwrite similar loops with the new template if the -** template is better. Loops may be overwritten if the following +** new template is better. Loops may be overwritten if the following ** conditions are met: ** ** (1) They have the same iTab. ** (2) They have the same iSortIdx. ** (3) The template has same or fewer dependencies than the current loop ** (4) The template has the same or lower cost than the current loop -** (5) The template uses more terms of the same index but has no additional -** dependencies */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p, *pNext = 0; + WhereLoop **ppPrev, *p; WhereInfo *pWInfo = pBuilder->pWInfo; sqlite3 *db = pWInfo->pParse->db; @@ -112557,64 +114510,23 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ return SQLITE_OK; } - /* Search for an existing WhereLoop to overwrite, or which takes - ** priority over pTemplate. + /* Look for an existing WhereLoop to replace with pTemplate */ - for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ - if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ - /* If either the iTab or iSortIdx values for two WhereLoop are different - ** then those WhereLoops need to be considered separately. Neither is - ** a candidate to replace the other. */ - continue; - } - /* In the current implementation, the rSetup value is either zero - ** or the cost of building an automatic index (NlogN) and the NlogN - ** is the same for compatible WhereLoops. */ - assert( p->rSetup==0 || pTemplate->rSetup==0 - || p->rSetup==pTemplate->rSetup ); - - /* whereLoopAddBtree() always generates and inserts the automatic index - ** case first. Hence compatible candidate WhereLoops never have a larger - ** rSetup. Call this SETUP-INVARIANT */ - assert( p->rSetup>=pTemplate->rSetup ); + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); - if( (p->prereq & pTemplate->prereq)==p->prereq - && p->rSetup<=pTemplate->rSetup - && p->rRun<=pTemplate->rRun - && p->nOut<=pTemplate->nOut - ){ - /* This branch taken when p is equal or better than pTemplate in - ** all of (1) dependencies (2) setup-cost, (3) run-cost, and - ** (4) number of output rows. */ - assert( p->rSetup==pTemplate->rSetup ); - if( p->prereq==pTemplate->prereq - && p->nLTermnLTerm - && (p->wsFlags & pTemplate->wsFlags & WHERE_INDEXED)!=0 - && (p->u.btree.pIndex==pTemplate->u.btree.pIndex - || pTemplate->rRun+p->nLTerm<=p->rRun+pTemplate->nLTerm) - ){ - /* Overwrite an existing WhereLoop with an similar one that uses - ** more terms of the index */ - pNext = p->pNextLoop; - break; - }else{ - /* pTemplate is not helpful. - ** Return without changing or adding anything */ - goto whereLoopInsert_noop; - } - } - if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rRun>=pTemplate->rRun - && p->nOut>=pTemplate->nOut - ){ - /* Overwrite an existing WhereLoop with a better one: one that is - ** better at one of (1) dependencies, (2) setup-cost, (3) run-cost - ** or (4) number of output rows, and is no worse in any of those - ** categories. */ - assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - pNext = p->pNextLoop; - break; + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-noop: "); + whereLoopPrint(pTemplate, pBuilder->pWC); } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; } /* If we reach this point it means that either p[] should be overwritten @@ -112632,13 +114544,33 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } #endif if( p==0 ){ - p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( ppTail==0 ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } } whereLoopXfer(db, p, pTemplate); - p->pNextLoop = pNext; - *ppPrev = p; if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ @@ -112646,16 +114578,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } } return SQLITE_OK; - - /* Jump here if the insert is a no-op */ -whereLoopInsert_noop: -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; } /* @@ -112685,13 +114607,30 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -112717,7 +114656,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -112735,15 +114673,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); - if( pNew->u.btree.nEq < pProbe->nKeyCol ){ - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; - }else{ - iCol = -1; - nRowEst = 0; - } + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; @@ -112753,18 +114685,23 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -112772,31 +114709,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -112806,84 +114752,120 @@ static int whereLoopAddBtreeIndex( /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); } - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 - || (pProbe->onError!=OE_None && nInMul==0 - && pNew->u.btree.nEq==pProbe->nKeyCol-1) - ){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); - pNew->wsFlags |= WHERE_ONEROW; + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + if( iCol>=0 && pProbe->onError==OE_None ){ + pNew->wsFlags |= WHERE_UNQ_WANTED; + }else{ + pNew->wsFlags |= WHERE_ONEROW; + } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ + }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ - assert( pNew->nOut==saved_nOut ); + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && iCol>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } } } -#endif + + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); + + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) + && pNew->u.btree.nEqnColumn ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } @@ -112965,6 +114947,37 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ @@ -112973,7 +114986,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -113007,12 +115020,14 @@ static int whereLoopAddBtree( Index *pFirst; /* First of real indices on the table */ memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; + sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + sPk.szIdxRow = pTab->szTabRow; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -113021,7 +115036,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -113050,6 +115065,7 @@ static int whereLoopAddBtree( ** approximately 7*N*log2(N) where N is the number of rows in ** the table being indexed. */ pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way ** of knowning how selective the index will ultimately be. It would @@ -113071,6 +115087,7 @@ static int whereLoopAddBtree( && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; @@ -113088,10 +115105,9 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; + ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -113118,19 +115134,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 1 + - (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) - ** which we will simplify to just N*log2(N) */ - pNew->rRun = rSize + rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -113301,8 +115314,8 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) - && pIdxInfo->orderByConsumed); + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -113334,7 +115347,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur, sPrev; + WhereOrSet sSum, sCur; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -113343,7 +115356,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew = pBuilder->pNew; memset(&sSum, 0, sizeof(sSum)); pItem = pWInfo->pTabList->a + pNew->iTab; - if( !HasRowid(pItem->pTab) ) return SQLITE_OK; iCur = pItem->iCursor; for(pTerm=pWC->a; pTermiSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs + ** of all sub-scans required by the OR-scan. However, due to rounding + ** errors, it may be that the cost of the OR-scan is equal to its + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that + ** this does not happen. Otherwise, for WHERE clauses such as the + ** following where there is an index on "y": + ** + ** WHERE likelihood(x=?, 0.99) OR y=? + ** + ** the planner may elect to "OR" together a full-table scan and an + ** index lookup. And other similarly odd results. */ + pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); @@ -113463,21 +115487,21 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 5th ** parameters) to see if it outputs rows in the requested ORDER BY -** (or GROUP BY) without requiring a separate sort operation. Return: +** (or GROUP BY) without requiring a separate sort operation. Return N: ** -** 0: ORDER BY is not satisfied. Sorting required -** 1: ORDER BY is satisfied. Omit sorting -** -1: Unknown at this time +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. ** ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY -** and DISTINT do not require rows to appear in any particular order as long +** and DISTINCT do not require rows to appear in any particular order as long ** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ -static int wherePathSatisfiesOrderBy( +static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ @@ -113533,14 +115557,6 @@ static int wherePathSatisfiesOrderBy( */ assert( pOrderBy!=0 ); - - /* Sortability of virtual tables is determined by the xBestIndex method - ** of the virtual table itself */ - if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ - testcase( nLoop>0 ); /* True when outer loops are one-row and match - ** no ORDER BY terms */ - return pLast->u.vtab.isOrdered; - } if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nOrderBy = pOrderBy->nExpr; @@ -113553,7 +115569,10 @@ static int wherePathSatisfiesOrderBy( for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; pLoop = iLoopaLoop[iLoop] : pLast; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; /* Mark off any ORDER BY term X that is a column in the table of @@ -113641,7 +115660,7 @@ static int wherePathSatisfiesOrderBy( } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and and mark that ORDER BY term off + ** of the index and mark that ORDER BY term off */ bOnce = 1; isMatch = 0; @@ -113662,23 +115681,23 @@ static int wherePathSatisfiesOrderBy( isMatch = 1; break; } + if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } if( isMatch ){ if( iColumn<0 ){ testcase( distinctColumns==0 ); distinctColumns = 1; } obSat |= MASKBIT(i); - if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ - /* Make sure the sort order is compatible in an ORDER BY clause. - ** Sort order is irrelevant for a GROUP BY clause. */ - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; - }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; - if( rev ) *pRevMask |= MASKBIT(iLoop); - revSet = 1; - } - } }else{ /* No match found */ if( j==0 || jmaskSelf; for(i=0; ia[i].pExpr; - if( (exprTableUsage(&pWInfo->sMaskSet, p)&~orderDistinctMask)==0 ){ + mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return 1; - if( !isOrderDistinct ) return 0; + if( obSat==obDone ) return (i8)nOrderBy; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } return -1; } + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; +} + #ifdef WHERETRACE_ENABLED /* For debugging use only: */ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ @@ -113724,7 +115782,6 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ } #endif - /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -113745,11 +115802,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ LogEst rCost; /* Cost of a path */ LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ - LogEst mxOut = 0; /* Maximum nOut value on the set of paths */ - LogEst rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -113765,7 +115821,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* TUNING: For simple queries, only the best path is tracked. ** For 2-way joins, the 5 best paths are followed. ** For joins of 3 or more tables, track the 10 best paths */ - mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); + mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver\n")); @@ -113791,16 +115847,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ - rSortCost = 0; if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - aFrom[0].isOrderedValid = 1; + aFrom[0].isOrdered = 0; + nOrderBy = 0; }else{ - /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the - ** number of output rows. The 48 is the expected size of a row to sort. - ** FIXME: compute a better estimate of the 48 multiplier based on the - ** result set expressions. */ - rSortCost = nRowEst + estLog(nRowEst); - WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); + aFrom[0].isOrdered = nLoop>0 ? -1 : 1; + nOrderBy = pWInfo->pOrderBy->nExpr; } /* Compute successively longer WherePaths using the previous generation @@ -113812,8 +115864,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; Bitmask revMask = 0; - u8 isOrderedValid = pFrom->isOrderedValid; - u8 isOrdered = pFrom->isOrdered; + i8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. @@ -113822,21 +115873,40 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; - if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, - iLoop, pWLoop, &revMask) ){ - case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ - isOrdered = 1; - isOrderedValid = 1; - break; - case 0: /* No. pFrom+pWLoop will require a separate sort */ - isOrdered = 0; - isOrderedValid = 1; - rCost = sqlite3LogEstAdd(rCost, rSortCost); - break; - default: /* Cannot tell yet. Try again on the next iteration */ - break; + iLoop, pWLoop, &revMask); + if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } + WHERETRACE(0x002, + ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", + rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost, + sqlite3LogEstAdd(rCost,rSortCost))); + rCost = sqlite3LogEstAdd(rCost, rSortCost); } }else{ revMask = pFrom->revLoop; @@ -113844,9 +115914,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Check to see if pWLoop should be added to the mxChoice best so far */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && pTo->isOrderedValid==isOrderedValid - && ((pTo->rCost<=rCost && pTo->nRow<=nOut) || - (pTo->rCost>=rCost && pTo->nRow>=nOut)) + && ((pTo->isOrdered^isOrdered)&80)==0 ){ testcase( jj==nTo-1 ); break; @@ -113858,7 +115926,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif continue; @@ -113876,20 +115944,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif }else{ - if( pTo->rCost<=rCost && pTo->nRow<=nOut ){ + if( pTo->rCost<=rCost ){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( "Skip %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif testcase( pTo->rCost==rCost ); @@ -113902,10 +115970,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ sqlite3DebugPrintf( "Update %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif } @@ -113914,18 +115982,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; - pTo->isOrderedValid = isOrderedValid; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ mxI = 0; mxCost = aTo[0].rCost; - mxOut = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost || (pTo->rCost==mxCost && pTo->nRow>mxOut) ){ + if( pTo->rCost>mxCost ){ mxCost = pTo->rCost; - mxOut = pTo->nRow; mxI = jj; } } @@ -113939,8 +116004,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); - if( pTo->isOrderedValid && pTo->isOrdered ){ + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); }else{ sqlite3DebugPrintf("\n"); @@ -113983,16 +116048,33 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ Bitmask notUsed; int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } } - if( pFrom->isOrdered ){ + if( pWInfo->pOrderBy ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } }else{ - pWInfo->bOBSat = 1; + pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + ){ + Bitmask notUsed = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used + ); + assert( pWInfo->sorted==0 ); + pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + } } + + pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ @@ -114074,7 +116156,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; - if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -114178,7 +116260,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ @@ -114200,6 +116282,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Variable initialization */ db = pParse->db; memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; sWLB.pOrderBy = pOrderBy; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via @@ -114244,7 +116330,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; pWInfo->pResultSet = pResultSet; - pWInfo->iBreak = sqlite3VdbeMakeLabel(v); + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = &pWInfo->sMaskSet; @@ -114263,7 +116349,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( initMaskSet(pMaskSet); whereClauseInit(&pWInfo->sWC, pWInfo); whereSplit(&pWInfo->sWC, pWhere, TK_AND); - sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. @@ -114279,7 +116364,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Special case: No FROM clause */ if( nTabList==0 ){ - if( pOrderBy ) pWInfo->bOBSat = 1; + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; if( wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -114325,22 +116410,6 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( goto whereBeginError; } - /* If the ORDER BY (or GROUP BY) clause contains references to general - ** expressions, then we won't be able to satisfy it using indices, so - ** go ahead and disable it now. - */ - if( pOrderBy && (wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ - for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOrderBy->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ){ - pWInfo->pOrderBy = pOrderBy = 0; - break; - }else if( pExpr->iColumn<0 ){ - break; - } - } - } - if( wctrlFlags & WHERE_WANT_DISTINCT ){ if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ @@ -114406,8 +116475,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( sqlite3WhereTrace ){ int ii; sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); - if( pWInfo->bOBSat ){ - sqlite3DebugPrintf(" ORDERBY=0x%llx", pWInfo->revMask); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); } switch( pWInfo->eDistinct ){ case WHERE_DISTINCT_UNIQUE: { @@ -114530,7 +116599,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( int op = OP_OpenRead; /* iIdxCur is always set if to a positive value if ONEPASS is possible */ assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); - if( pWInfo->okOnePass ){ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) + && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + ){ + /* This is one term of an OR-optimization using the PRIMARY KEY of a + ** WITHOUT ROWID table. No need for a separate index */ + iIndexCur = pLevel->iTabCur; + op = 0; + }else if( pWInfo->okOnePass ){ Index *pJ = pTabItem->pTab->pIndex; iIndexCur = iIdxCur; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); @@ -114542,17 +116618,20 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->aiCurOnePass[1] = iIndexCur; }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; } pLevel->iIdxCur = iIndexCur; assert( pIx->pSchema==pTab->pSchema ); assert( iIndexCur>=0 ); - sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIx); - VdbeComment((v, "%s", pIx->zName)); + if( op ){ + sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + VdbeComment((v, "%s", pIx->zName)); + } } - sqlite3CodeVerifySchema(pParse, iDb); + if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); @@ -114614,8 +116693,12 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); } if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; @@ -114624,6 +116707,9 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } sqlite3DbFree(db, pLevel->u.in.aInLoop); @@ -114636,7 +116722,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } if( pLevel->iLeftJoin ){ - addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ @@ -114663,12 +116749,38 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + int k, last; + VdbeOp *pOp; Index *pIdx = 0; struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; Table *pTab = pTabItem->pTab; assert( pTab!=0 ); pLoop = pLevel->pWLoop; + /* For a co-routine, change all OP_Column references to the table of + ** the co-routine into OP_SCopy of result contained in a register. + ** OP_Rowid becomes OP_Null. + */ + if( pTabItem->viaCoroutine && !db->mallocFailed ){ + last = sqlite3VdbeCurrentAddr(v); + k = pLevel->addrBody; + pOp = sqlite3VdbeGetOp(v, k); + for(; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + pTabItem->regResult; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + pOp->opcode = OP_Null; + pOp->p1 = 0; + pOp->p3 = 0; + } + } + continue; + } + /* Close all of the cursors that were opened by sqlite3WhereBegin. ** Except, do not close cursors that will be reused by the OR optimization ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors @@ -114707,9 +116819,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pIdx = pLevel->u.pCovidx; } if( pIdx && !db->mallocFailed ){ - int k, last; - VdbeOp *pOp; - last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); @@ -117123,13 +119232,26 @@ static void yy_reduce( } break; case 112: /* select ::= with selectnowith */ -{ - if( yymsp[0].minor.yy3 ){ - yymsp[0].minor.yy3->pWith = yymsp[-1].minor.yy59; +{ + Select *p = yymsp[0].minor.yy3, *pNext, *pLoop; + if( p ){ + int cnt = 0, mxSelect; + p->pWith = yymsp[-1].minor.yy59; + if( p->pPrior ){ + pNext = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + } + mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; + if( mxSelect && cnt>mxSelect ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } }else{ sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy59); } - yygotominor.yy3 = yymsp[0].minor.yy3; + yygotominor.yy3 = p; } break; case 113: /* selectnowith ::= oneselect */ @@ -117138,14 +119260,22 @@ static void yy_reduce( break; case 114: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - if( yymsp[0].minor.yy3 ){ - yymsp[0].minor.yy3->op = (u8)yymsp[-1].minor.yy328; - yymsp[0].minor.yy3->pPrior = yymsp[-2].minor.yy3; + Select *pRhs = yymsp[0].minor.yy3; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy328; + pRhs->pPrior = yymsp[-2].minor.yy3; if( yymsp[-1].minor.yy328!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy3); } - yygotominor.yy3 = yymsp[0].minor.yy3; + yygotominor.yy3 = pRhs; } break; case 116: /* multiselect_op ::= UNION ALL */ @@ -117586,6 +119716,33 @@ static void yy_reduce( */ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy328]); sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy346.pExpr); + }else if( yymsp[-1].minor.yy14->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to something + ** like this: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. + */ + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; + } + yygotominor.yy346.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy328 ? TK_NE : TK_EQ, yymsp[-4].minor.yy346.pExpr, pRHS, 0); }else{ yygotominor.yy346.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy346.pExpr, 0, 0); if( yygotominor.yy346.pExpr ){ @@ -118786,6 +120943,12 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); testcase( z[0]=='9' ); *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif for(i=0; sqlite3Isdigit(z[i]); i++){} #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' ){ @@ -120206,6 +122369,7 @@ static void disconnectAllVtab(sqlite3 *db){ } } } + sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else UNUSED_PARAMETER(db); @@ -121506,8 +123670,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>62 -# error SQLITE_MAX_ATTACHED must be between 0 and 62 +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 @@ -122519,6 +124683,28 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** @@ -122610,6 +124796,22 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) ** ** Set the nReserve size to N for the main database on the database @@ -122713,6 +124915,21 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,int,u8,u8); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ @@ -122761,7 +124978,7 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64( ){ const char *z = sqlite3_uri_parameter(zFilename, zParam); sqlite3_int64 v; - if( z && sqlite3Atoi64(z, &v, sqlite3Strlen30(z), SQLITE_UTF8)==SQLITE_OK ){ + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ bDflt = v; } return bDflt; @@ -122797,7 +125014,7 @@ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ */ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ Btree *pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3PagerIsreadonly(sqlite3BtreePager(pBt)) : -1; + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } /************** End of main.c ************************************************/ @@ -123917,20 +126134,20 @@ struct Fts3Table { sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ @@ -124292,7 +126509,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); @@ -125345,7 +127562,7 @@ static int fts3InitVtab( p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; @@ -125388,7 +127605,9 @@ static int fts3InitVtab( int n = (int)strlen(p->azColumn[iCol]); for(i=0; iazColumn[iCol], zNot, n) ){ + if( zNot && n==(int)strlen(zNot) + && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) + ){ p->abNotindexed[iCol] = 1; sqlite3_free(zNot); azNotindexed[i] = 0; @@ -125422,10 +127641,7 @@ static int fts3InitVtab( ** addition of a %_stat table so that it can use incremental merge. */ if( !isFts4 && !isCreate ){ - int rc2 = SQLITE_OK; - fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", - p->zDb, p->zName); - if( rc2==SQLITE_OK ) p->bHasStat = 1; + p->bHasStat = 2; } /* Figure out the page-size for the database. This is required in order to @@ -127317,7 +129533,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ @@ -127325,14 +129544,41 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; } /* -** Implementation of xBegin() method. This is a no-op. +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; @@ -127343,7 +129589,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); p->nLeafAdd = 0; - return SQLITE_OK; + return fts3SetHasStat(p); } /* @@ -127592,6 +129838,10 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + /* As it happens, the pending terms table is always empty here. This is ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction ** always opens a savepoint transaction. And the xSavepoint() method @@ -127599,7 +129849,9 @@ static int fts3RenameMethod( ** PendingTermsFlush() in in case that changes. */ assert( p->nPendingData==0 ); - rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, @@ -127727,7 +129979,7 @@ static void hashDestroy(void *p){ */ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif #ifdef SQLITE_ENABLE_ICU @@ -127745,7 +129997,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; #endif @@ -127754,7 +130006,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE sqlite3Fts3UnicodeTokenizer(&pUnicode); #endif @@ -127782,7 +130034,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU @@ -130515,40 +132767,23 @@ static int getNextToken( int rc; sqlite3_tokenizer_cursor *pCursor; Fts3Expr *pRet = 0; - int nConsumed = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, n, &pCursor); + *pnConsumed = i; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; int nByte; /* total space to allocate */ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - - if( (rc==SQLITE_OK || rc==SQLITE_DONE) && sqlite3_fts3_enable_parentheses ){ - int i; - if( rc==SQLITE_DONE ) iStart = n; - for(i=0; inNest++; - rc = fts3ExprParse(pParse, &z[i+1], n-i-1, &pRet, &nConsumed); - if( rc==SQLITE_OK && !pRet ){ - rc = SQLITE_DONE; - } - nConsumed = (int)(i + 1 + nConsumed); - break; - } - - if( z[i]==')' ){ - rc = SQLITE_DONE; - pParse->nNest--; - nConsumed = i+1; - break; - } - } - } - - if( nConsumed==0 && rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ @@ -130582,13 +132817,14 @@ static int getNextToken( } } - nConsumed = iEnd; + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } pModule->xClose(pCursor); } - *pnConsumed = nConsumed; *ppExpr = pRet; return rc; } @@ -130838,6 +133074,21 @@ static int getNextNode( return getNextString(pParse, &zInput[1], ii-1, ppExpr); } + if( sqlite3_fts3_enable_parentheses ){ + if( *zInput=='(' ){ + int nConsumed = 0; + pParse->nNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer @@ -130956,96 +133207,100 @@ static int fts3ExprParse( while( rc==SQLITE_OK ){ Fts3Expr *p = 0; int nByte = 0; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); if( rc==SQLITE_OK ){ - int isPhrase; - - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + if( p ){ + int isPhrase; - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } - - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } }else{ - pRet = p; + insertBinaryOperator(&pRet, pPrev, p); } - }else{ - insertBinaryOperator(&pRet, pPrev, p); + isRequirePhrase = !isPhrase; } - isRequirePhrase = !isPhrase; + pPrev = p; } assert( nByte>0 ); } assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); nIn -= nByte; zIn += nByte; - pPrev = p; } if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ @@ -134033,6 +136288,7 @@ struct SegmentWriter { int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* @@ -134108,6 +136364,10 @@ struct SegmentNode { #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -134209,7 +136469,18 @@ static int fts3SqlStmt( /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -135750,6 +138021,7 @@ static int fts3WriteSegdir( sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ @@ -135760,7 +138032,13 @@ static int fts3WriteSegdir( sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -136086,6 +138364,9 @@ static int fts3SegWriterAdd( nDoclist; /* Doclist data */ } + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ @@ -136157,13 +138438,13 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; @@ -136247,6 +138528,37 @@ static int fts3SegmentMaxLevel( return sqlite3_reset(pStmt); } +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents @@ -136782,6 +139094,140 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( } } +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -136806,6 +139252,7 @@ static int fts3SegmentMerge( Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING @@ -136817,6 +139264,11 @@ static int fts3SegmentMerge( rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically @@ -136826,21 +139278,21 @@ static int fts3SegmentMerge( rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + } + } + } finished: fts3SegWriterFree(pWriter); @@ -136875,7 +139334,7 @@ static int fts3SegmentMerge( /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; @@ -136891,14 +139350,19 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } rc = sqlite3_reset(pStmt); } } @@ -137266,6 +139730,8 @@ struct IncrmergeWriter { int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; @@ -137604,8 +140070,8 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; @@ -137704,6 +140170,7 @@ static void fts3IncrmergeRelease( pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } @@ -137805,7 +140272,11 @@ static int fts3IncrmergeLoad( if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ @@ -138406,11 +140877,11 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ @@ -138435,6 +140906,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. @@ -138488,6 +140960,19 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } @@ -138495,16 +140980,12 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ @@ -138526,7 +141007,13 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); @@ -138613,7 +141100,10 @@ static int fts3DoAutoincrmerge( ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); @@ -138622,7 +141112,7 @@ static int fts3DoAutoincrmerge( rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; @@ -138779,34 +141269,36 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int iCol; for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); - int nText = sqlite3_column_bytes(pStmt, iCol+1); - sqlite3_tokenizer_cursor *pT = 0; - - rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ - - rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); - if( rc==SQLITE_OK ){ - int i; - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, nToken, iLang, 0, iDocid, iCol, iPos - ); - for(i=1; inIndex; i++){ - if( p->aIndex[i].nPrefix<=nToken ){ - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos - ); + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + int nText = sqlite3_column_bytes(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } } } } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - if( pT ) pModule->xClose(pT); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; } } @@ -139111,6 +141603,10 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + assert( p->pSegments==0 ); assert( nArg==1 /* DELETE operations */ @@ -140798,7 +143294,7 @@ SQLITE_PRIVATE void sqlite3Fts3Matchinfo( ** Implementation of the "unicode" full-text-search tokenizer. */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -141014,7 +143510,7 @@ static int unicodeCreate( for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; @@ -141146,11 +143642,11 @@ static int unicodeNext( ); /* Set the output variables and return. */ - pCsr->iOff = (z - pCsr->aInput); + pCsr->iOff = (int)(z - pCsr->aInput); *paToken = pCsr->zToken; - *pnToken = zOut - pCsr->zToken; - *piStart = (zStart - pCsr->aInput); - *piEnd = (zEnd - pCsr->aInput); + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); *piPos = pCsr->iToken++; return SQLITE_OK; } @@ -141173,7 +143669,7 @@ SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const * } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ /************** End of fts3_unicode.c ****************************************/ /************** Begin file fts3_unicode2.c ***********************************/ @@ -141194,7 +143690,7 @@ SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const * ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -#if defined(SQLITE_ENABLE_FTS4_UNICODE61) +#ifndef SQLITE_DISABLE_FTS3_UNICODE #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) /* #include */ @@ -141541,7 +144037,7 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ return ret; } #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ -#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ /************** End of fts3_unicode2.c ***************************************/ /************** Begin file rtree.c *******************************************/ @@ -141601,48 +144097,6 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) -/* -** This file contains an implementation of a couple of different variants -** of the r-tree algorithm. See the README file for further details. The -** same data-structure is used for all, but the algorithms for insert and -** delete operations vary. The variants used are selected at compile time -** by defining the following symbols: -*/ - -/* Either, both or none of the following may be set to activate -** r*tree variant algorithms. -*/ -#define VARIANT_RSTARTREE_CHOOSESUBTREE 0 -#define VARIANT_RSTARTREE_REINSERT 1 - -/* -** Exactly one of the following must be set to 1. -*/ -#define VARIANT_GUTTMAN_QUADRATIC_SPLIT 0 -#define VARIANT_GUTTMAN_LINEAR_SPLIT 0 -#define VARIANT_RSTARTREE_SPLIT 1 - -#define VARIANT_GUTTMAN_SPLIT \ - (VARIANT_GUTTMAN_LINEAR_SPLIT||VARIANT_GUTTMAN_QUADRATIC_SPLIT) - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT - #define PickNext QuadraticPickNext - #define PickSeeds QuadraticPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_GUTTMAN_LINEAR_SPLIT - #define PickNext LinearPickNext - #define PickSeeds LinearPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_RSTARTREE_SPLIT - #define AssignCells splitNodeStartree -#endif - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #else @@ -141650,11 +144104,13 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ /* #include */ /* #include */ +/* #include */ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; +typedef unsigned short u16; typedef unsigned int u32; #endif @@ -141672,6 +144128,7 @@ typedef struct RtreeConstraint RtreeConstraint; typedef struct RtreeMatchArg RtreeMatchArg; typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -141680,7 +144137,7 @@ typedef union RtreeCoord RtreeCoord; ** ever contain very many entries, so a fixed number of buckets is ** used. */ -#define HASHSIZE 128 +#define HASHSIZE 97 /* The xBestIndex method of this virtual table requires an estimate of ** the number of rows in the virtual table to calculate the costs of @@ -141696,15 +144153,15 @@ typedef union RtreeCoord RtreeCoord; ** An rtree virtual-table object. */ struct Rtree { - sqlite3_vtab base; + sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ - int nDim; /* Number of dimensions */ - int nBytesPerCell; /* Bytes consumed per cell */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ @@ -141731,10 +144188,10 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - int eCoordType; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; -/* Possible values for eCoordType: */ +/* Possible values for Rtree.eCoordType: */ #define RTREE_COORD_REAL32 0 #define RTREE_COORD_INT32 1 @@ -141746,11 +144203,30 @@ struct Rtree { #ifdef SQLITE_RTREE_INT_ONLY typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 #else typedef double RtreeDValue; /* High accuracy coordinate */ typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 #endif +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + /* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: @@ -141773,21 +144249,44 @@ struct Rtree { */ #define RTREE_MAX_DEPTH 40 + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + /* ** An rtree cursor object. */ struct RtreeCursor { - sqlite3_vtab_cursor base; - RtreeNode *pNode; /* Node cursor is currently pointing at */ - int iCell; /* Index of current cell in pNode */ + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ }; +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ union RtreeCoord { - RtreeValue f; - int i; + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ }; /* @@ -141812,38 +144311,67 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - RtreeDValue rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 -#define RTREE_MATCH 0x46 +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + /* ** An rtree structure node. */ struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; - int nRef; - int isDirty; - u8 *zData; - RtreeNode *pNext; /* Next node in this hash chain */ + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ }; + +/* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) /* -** Structure to store a deserialized rtree record. +** A single cell from a node, deserialized */ struct RtreeCell { - i64 iRowid; - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; }; @@ -141855,29 +144383,16 @@ struct RtreeCell { #define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** An instance of this structure must be supplied as a blob argument to -** the right-hand-side of an SQL MATCH operator used to constrain an -** r-tree query. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *); - void *pContext; - int nParam; - RtreeDValue aParam[1]; -}; - -/* -** When a geometry callback is created (see sqlite3_rtree_geometry_callback), -** a single instance of the following structure is allocated. It is used -** as the context for the user-function created by by s_r_g_c(). The object -** is eventually deleted by the destructor mechanism provided by -** sqlite3_create_function_v2() (which is called by s_r_g_c() to create -** the geometry callback function). -*/ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - void *pContext; + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; #ifndef MAX @@ -141971,10 +144486,7 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** in the Rtree.aHash table. */ static int nodeHash(i64 iNode){ - return ( - (iNode>>56) ^ (iNode>>48) ^ (iNode>>40) ^ (iNode>>32) ^ - (iNode>>24) ^ (iNode>>16) ^ (iNode>> 8) ^ (iNode>> 0) - ) % HASHSIZE; + return iNode % HASHSIZE; } /* @@ -142034,8 +144546,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ /* ** Obtain a reference to an r-tree node. */ -static int -nodeAcquire( +static int nodeAcquire( Rtree *pRtree, /* R-tree structure */ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ @@ -142124,10 +144635,10 @@ nodeAcquire( ** Overwrite cell iCell of node pNode with the contents of pCell. */ static void nodeOverwriteCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ ){ int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -142139,7 +144650,7 @@ static void nodeOverwriteCell( } /* -** Remove cell the cell with index iCell from node pNode. +** Remove the cell with index iCell from node pNode. */ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -142156,11 +144667,10 @@ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ ** ** If there is not enough free space in pNode, return SQLITE_FULL. */ -static int -nodeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ int nCell; /* Current number of cells in pNode */ int nMaxCell; /* Maximum number of cells for pNode */ @@ -142181,8 +144691,7 @@ nodeInsertCell( /* ** If the node is dirty, write it out to the database. */ -static int -nodeWrite(Rtree *pRtree, RtreeNode *pNode){ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode->isDirty ){ sqlite3_stmt *p = pRtree->pWriteNode; @@ -142207,8 +144716,7 @@ nodeWrite(Rtree *pRtree, RtreeNode *pNode){ ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. */ -static int -nodeRelease(Rtree *pRtree, RtreeNode *pNode){ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); @@ -142236,9 +144744,9 @@ nodeRelease(Rtree *pRtree, RtreeNode *pNode){ ** an internal node, then the 64-bit integer is a child page number. */ static i64 nodeGetRowid( - Rtree *pRtree, - RtreeNode *pNode, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); @@ -142248,11 +144756,11 @@ static i64 nodeGetRowid( ** Return coordinate iCoord from cell iCell in node pNode. */ static void nodeGetCoord( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - int iCoord, - RtreeCoord *pCoord /* Space to write result to */ + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ ){ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -142262,15 +144770,20 @@ static void nodeGetCoord( ** to by pCell with the results. */ static void nodeGetCell( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - RtreeCell *pCell -){ - int ii; + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - for(ii=0; iinDim*2; ii++){ - nodeGetCoord(pRtree, pNode, iCell, ii, &pCell->aCoord[ii]); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDataaConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ - sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; - if( pGeom ){ - if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); - sqlite3_free(pGeom); + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); } } sqlite3_free(pCsr->aConstraint); @@ -142412,12 +144925,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int rc; + int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; freeCursorConstraints(pCsr); - rc = nodeRelease(pRtree, pCsr->pNode); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); - return rc; + return SQLITE_OK; } /* @@ -142428,194 +144942,164 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; - return (pCsr->pNode==0); + return pCsr->atEOF; +} + +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. +*/ +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } +#endif /* -** The r-tree constraint passed as the second argument to this function is -** guaranteed to be a MATCH constraint. +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. */ -static int testRtreeGeom( - Rtree *pRtree, /* R-Tree object */ - RtreeConstraint *pConstraint, /* MATCH constraint to test */ - RtreeCell *pCell, /* Cell to test */ - int *pbRes /* OUT: Test result */ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ ){ - int i; - RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2]; - int nCoord = pRtree->nDim*2; + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - assert( pConstraint->op==RTREE_MATCH ); - assert( pConstraint->pGeom ); + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); - for(i=0; iaCoord[i]); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); } - return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); -} - -/* -** Cursor pCursor currently points to a cell in a non-leaf page. -** Set *pbEof to true if the sub-tree headed by the cell is filtered -** (excluded) by the constraints in the pCursor->aConstraint[] -** array, or false otherwise. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -*/ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - int bRes = 0; - int rc = SQLITE_OK; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; bRes==0 && iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - - switch( p->op ){ - case RTREE_LE: case RTREE_LT: - bRes = p->rValuerValue>cell_max; - break; - - case RTREE_EQ: - bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &bRes); - bRes = !bRes; - break; - } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - *pbEof = bRes; return rc; } /* -** Test if the cell that cursor pCursor currently points to -** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array. If so, set *pbEof to true before -** returning. If the cell is not filtered (excluded) by the constraints, -** set pbEof to zero. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -** -** This function assumes that the cell is part of a leaf node. +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. */ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - *pbEof = 0; +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]); - int res; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - switch( p->op ){ - case RTREE_LE: res = (coord<=p->rValue); break; - case RTREE_LT: res = (coordrValue); break; - case RTREE_GE: res = (coord>=p->rValue); break; - case RTREE_GT: res = (coord>p->rValue); break; - case RTREE_EQ: res = (coord==p->rValue); break; - default: { - int rc; - assert( p->op==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &res); - if( rc!=SQLITE_OK ){ - return rc; - } - break; - } - } + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); - if( !res ){ - *pbEof = 1; - return SQLITE_OK; - } - } + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ - return SQLITE_OK; + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; } /* -** Cursor pCursor currently points at a node that heads a sub-tree of -** height iHeight (if iHeight==0, then the node is a leaf). Descend -** to point to the left-most cell of the sub-tree that matches the -** configured constraints. +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. */ -static int descendToCell( - Rtree *pRtree, - RtreeCursor *pCursor, - int iHeight, - int *pEof /* OUT: Set to true if cannot descend */ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ ){ - int isEof; - int rc; - int ii; - RtreeNode *pChild; - sqlite3_int64 iRowid; + RtreeDValue xN; /* Coordinate value converted to a double */ - RtreeNode *pSavedNode = pCursor->pNode; - int iSavedCell = pCursor->iCell; - - assert( iHeight>=0 ); - - if( iHeight==0 ){ - rc = testRtreeEntry(pRtree, pCursor, &isEof); - }else{ - rc = testRtreeCell(pRtree, pCursor, &isEof); - } - if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - goto descend_to_cell_out; - } - - iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); - rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - - nodeRelease(pRtree, pCursor->pNode); - pCursor->pNode = pChild; - isEof = 1; - for(ii=0; isEof && iiiCell = ii; - rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - } - - if( isEof ){ - assert( pCursor->pNode==pChild ); - nodeReference(pSavedNode); - nodeRelease(pRtree, pChild); - pCursor->pNode = pSavedNode; - pCursor->iCell = iSavedCell; + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; } - -descend_to_cell_out: - *pEof = isEof; - return rc; + *peWithin = NOT_WITHIN; } /* @@ -142630,6 +145114,7 @@ static int nodeRowidIndex( ){ int ii; int nCell = NCELL(pNode); + assert( nCell<200 ); for(ii=0; iipVtab); - RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - int rc = SQLITE_OK; +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB +){ + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} - /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is - ** already at EOF. It is against the rules to call the xNext() method of - ** a cursor that has already reached EOF. - */ - assert( pCsr->pNode ); - - if( pCsr->iStrategy==1 ){ - /* This "scan" is a direct lookup by rowid. There is no next entry. */ - nodeRelease(pRtree, pCsr->pNode); - pCsr->pNode = 0; - }else{ - /* Move to the next entry that matches the configured constraints. */ - int iHeight = 0; - while( pCsr->pNode ){ - RtreeNode *pNode = pCsr->pNode; - int nCell = NCELL(pNode); - for(pCsr->iCell++; pCsr->iCelliCell++){ - int isEof; - rc = descendToCell(pRtree, pCsr, iHeight, &isEof); - if( rc!=SQLITE_OK || !isEof ){ - return rc; +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; } } - pCsr->pNode = pNode->pParent; - rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); - if( rc!=SQLITE_OK ){ - return rc; + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; } - nodeReference(pCsr->pNode); - nodeRelease(pRtree, pNode); - iHeight++; + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); } } + pCur->atEOF = p==0; + return SQLITE_OK; +} +/* +** Rtree virtual table module xNext method. +*/ +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); return rc; } @@ -142701,13 +145440,14 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - - assert(pCsr->pNode); - *pRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - - return SQLITE_OK; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; } /* @@ -142716,13 +145456,18 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ Rtree *pRtree = (Rtree *)cur->pVtab; RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; if( i==0 ){ - i64 iRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - sqlite3_result_int64(ctx, iRowid); + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - RtreeCoord c; - nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); @@ -142733,7 +145478,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ sqlite3_result_int(ctx, c.i); } } - return SQLITE_OK; } @@ -142744,12 +145488,18 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf ** to zero and return an SQLite error code. */ -static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ int rc; *ppLeaf = 0; sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); sqlite3_reset(pRtree->pReadRowid); }else{ @@ -142765,9 +145515,10 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *p; - sqlite3_rtree_geometry *pGeom; - int nBlob; + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ /* Check that value is actually a blob. */ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; @@ -142780,27 +145531,29 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_ERROR; } - pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( - sizeof(sqlite3_rtree_geometry) + nBlob - ); - if( !pGeom ) return SQLITE_NOMEM; - memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); - p = (RtreeMatchArg *)&pGeom[1]; + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; - memcpy(p, sqlite3_value_blob(pValue), nBlob); - if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue)) - ){ - sqlite3_free(pGeom); + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); return SQLITE_ERROR; } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; - pGeom->pContext = p->pContext; - pGeom->nParam = p->nParam; - pGeom->aParam = p->aParam; - - pCons->xGeom = p->xGeom; - pCons->pGeom = pGeom; + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; return SQLITE_OK; } @@ -142814,10 +145567,10 @@ static int rtreeFilter( ){ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; int ii; int rc = SQLITE_OK; + int iCell = 0; rtreeReference(pRtree); @@ -142827,31 +145580,42 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); - rc = findLeafNode(pRtree, iRowid, &pLeaf); - pCsr->pNode = pLeaf; - if( pLeaf ){ - assert( rc==SQLITE_OK ); - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ - if( argc>0 ){ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); assert( (idxStr==0 && argc==0) || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'a'; - if( p->op==RTREE_MATCH ){ + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ /* A MATCH operator. The right-hand-side must be a blob that ** can be cast into an RtreeMatchArg object. One created using ** an sqlite3_rtree_geometry_callback() SQL user function. @@ -142860,41 +145624,35 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ #ifdef SQLITE_RTREE_INT_ONLY - p->rValue = sqlite3_value_int64(argv[ii]); + p->u.rValue = sqlite3_value_int64(argv[ii]); #else - p->rValue = sqlite3_value_double(argv[ii]); + p->u.rValue = sqlite3_value_double(argv[ii]); #endif } } } } - - if( rc==SQLITE_OK ){ - pCsr->pNode = 0; - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - } if( rc==SQLITE_OK ){ - int isEof = 1; - int nCell = NCELL(pRoot); - pCsr->pNode = pRoot; - for(pCsr->iCell=0; rc==SQLITE_OK && pCsr->iCelliCell++){ - assert( pCsr->pNode==pRoot ); - rc = descendToCell(pRtree, pCsr, pRtree->iDepth, &isEof); - if( !isEof ){ - break; - } - } - if( rc==SQLITE_OK && isEof ){ - assert( pCsr->pNode==pRoot ); - nodeRelease(pRtree, pRoot); - pCsr->pNode = 0; - } - assert( rc!=SQLITE_OK || !pCsr->pNode || pCsr->iCellpNode) ); + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } + nodeRelease(pRtree, pRoot); rtreeRelease(pRtree); return rc; } @@ -142996,7 +145754,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -143089,62 +145847,32 @@ static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ return (cellArea(pRtree, &cell)-area); } -#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, - int nCell, - int iExclude + int nCell ){ int ii; - RtreeDValue overlap = 0.0; + RtreeDValue overlap = RTREE_ZERO; for(ii=0; iinDim*2); jj+=2){ - RtreeDValue x1, x2; - - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - - if( x2nDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-1) ){ - int jj; - aCell = sqlite3_malloc(sizeof(RtreeCell)*nCell); - if( !aCell ){ - rc = SQLITE_NOMEM; - nodeRelease(pRtree, pNode); - pNode = 0; - continue; - } - for(jj=0; jjiDepth-1) ){ - overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); - }else{ - overlap = 0.0; - } - if( (iCell==0) - || (overlapnDim; i++){ - RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]); - RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]); - RtreeDValue x3 = x1; - RtreeDValue x4 = x2; - int jj; - - int iCellLeft = 0; - int iCellRight = 0; - - for(jj=1; jjx4 ) x4 = right; - if( left>x3 ){ - x3 = left; - iCellRight = jj; - } - if( rightmaxNormalInnerWidth ){ - iLeftSeed = iCellLeft; - iRightSeed = iCellRight; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_LINEAR_SPLIT */ - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT -/* -** Implementation of the quadratic variant of the PickNext() function from -** Guttman[84]. -*/ -static RtreeCell *QuadraticPickNext( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeCell *pLeftBox, - RtreeCell *pRightBox, - int *aiUsed -){ - #define FABS(a) ((a)<0.0?-1.0*(a):(a)) - - int iSelect = -1; - RtreeDValue fDiff; - int ii; - for(ii=0; iifDiff ){ - fDiff = diff; - iSelect = ii; - } - } - } - aiUsed[iSelect] = 1; - return &aCell[iSelect]; -} - -/* -** Implementation of the quadratic variant of the PickSeeds() function from -** Guttman[84]. -*/ -static void QuadraticPickSeeds( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - int *piLeftSeed, - int *piRightSeed -){ - int ii; - int jj; - - int iLeftSeed = 0; - int iRightSeed = 1; - RtreeDValue fWaste = 0.0; - - for(ii=0; iifWaste ){ - iLeftSeed = ii; - iRightSeed = jj; - fWaste = waste; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_QUADRATIC_SPLIT */ /* ** Arguments aIdx, aDistance and aSpare all point to arrays of size @@ -143585,7 +146127,6 @@ static void SortByDimension( } } -#if VARIANT_RSTARTREE_SPLIT /* ** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. */ @@ -143604,7 +146145,7 @@ static int splitNodeStartree( int iBestDim = 0; int iBestSplit = 0; - RtreeDValue fBestMargin = 0.0; + RtreeDValue fBestMargin = RTREE_ZERO; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -143625,9 +146166,9 @@ static int splitNodeStartree( } for(ii=0; iinDim; ii++){ - RtreeDValue margin = 0.0; - RtreeDValue fBestOverlap = 0.0; - RtreeDValue fBestArea = 0.0; + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; int iBestLeft = 0; int nLeft; @@ -143653,7 +146194,7 @@ static int splitNodeStartree( } margin += cellMargin(pRtree, &left); margin += cellMargin(pRtree, &right); - overlap = cellOverlap(pRtree, &left, &right, 1, -1); + overlap = cellOverlap(pRtree, &left, &right, 1); area = cellArea(pRtree, &left) + cellArea(pRtree, &right); if( (nLeft==RTREE_MINCELLS(pRtree)) || (overlap0; i--){ - RtreeCell *pNext; - pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - RtreeDValue diff = - cellGrowth(pRtree, pBboxLeft, pNext) - - cellGrowth(pRtree, pBboxRight, pNext) - ; - if( (RTREE_MINCELLS(pRtree)-NCELL(pRight)==i) - || (diff>0.0 && (RTREE_MINCELLS(pRtree)-NCELL(pLeft)!=i)) - ){ - nodeInsertCell(pRtree, pRight, pNext); - cellUnion(pRtree, pBboxRight, pNext); - }else{ - nodeInsertCell(pRtree, pLeft, pNext); - cellUnion(pRtree, pBboxLeft, pNext); - } - } - - sqlite3_free(aiUsed); - return SQLITE_OK; -} -#endif static int updateMapping( Rtree *pRtree, @@ -143819,7 +146304,8 @@ static int SplitNode( memset(pLeft->zData, 0, pRtree->iNodeSize); memset(pRight->zData, 0, pRtree->iNodeSize); - rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } @@ -144102,7 +146588,7 @@ static int Reinsert( } for(ii=0; iinDim; iDim++){ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - DCOORD(aCell[ii].aCoord[iDim*2])); @@ -144168,16 +146654,12 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ -#if VARIANT_RSTARTREE_REINSERT if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ pRtree->iReinsertHeight = iHeight; rc = Reinsert(pRtree, pNode, pCell, iHeight); } -#else - rc = SplitNode(pRtree, pNode, pCell, iHeight); -#endif }else{ rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ @@ -144247,7 +146729,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ ** about to be deleted. */ if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } /* Delete the cell in question from the leaf node. */ @@ -144492,26 +146974,32 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ ** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. */ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ - const char *zSql = "SELECT stat FROM sqlite_stat1 WHERE tbl= ? || '_rowid'"; + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; sqlite3_stmt *p; int rc; i64 nRow = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_text(p, 1, pRtree->zName, -1, SQLITE_STATIC); - if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); - rc = sqlite3_finalize(p); - }else if( rc!=SQLITE_NOMEM ){ - rc = SQLITE_OK; - } + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; + } - if( rc==SQLITE_OK ){ - if( nRow==0 ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - }else{ - pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } } + sqlite3_free(zSql); } return rc; @@ -144578,7 +147066,8 @@ static int rtreeSqlInit( char *zCreate = sqlite3_mprintf( "CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY, parentnode INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" "INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize ); @@ -144780,6 +147269,8 @@ static int rtreeInit( if( rc==SQLITE_OK ){ *ppVtab = (sqlite3_vtab *)pRtree; }else{ + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); rtreeRelease(pRtree); } return rc; @@ -144790,10 +147281,10 @@ static int rtreeInit( ** Implementation of a scalar function that decodes r-tree nodes to ** human readable strings. This can be used for debugging and analysis. ** -** The scalar function takes two arguments, a blob of data containing -** an r-tree node, and the number of dimensions the r-tree indexes. -** For a two-dimensional r-tree structure called "rt", to deserialize -** all nodes, a statement like: +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** ** SELECT rtreenode(2, data) FROM rt_node; ** @@ -144826,7 +147317,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nCell = (int)strlen(zCell); for(jj=0; jjxDestructor ) pInfo->xDestructor(pInfo->pContext); sqlite3_free(p); } /* -** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite -** scalar user function. This C function is the callback used for all such -** registered SQL functions. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. ** -** The scalar user functions return a blob that is interpreted by r-tree -** table MATCH operators. +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); @@ -144918,8 +147427,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ }else{ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->xGeom = pGeomCtx->xGeom; - pBlob->pContext = pGeomCtx->pContext; + pBlob->cb = pGeomCtx[0]; pBlob->nParam = nArg; for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); #endif } - sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); } } @@ -144936,10 +147444,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ ** Register a new geometry function for use with the r-tree MATCH operator. */ SQLITE_API int sqlite3_rtree_geometry_callback( - sqlite3 *db, - const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *), - void *pContext + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ @@ -144947,12 +147455,36 @@ SQLITE_API int sqlite3_rtree_geometry_callback( pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); if( !pGeomCtx ) return SQLITE_NOMEM; pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - - /* Create the new user-function. Register a destructor function to delete - ** the context object when it is no longer required. */ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } diff --git a/EasyTTOra/INSTALL.md b/EasyTTOra/INSTALL.md index 5cfa76f..8ed967f 100644 --- a/EasyTTOra/INSTALL.md +++ b/EasyTTOra/INSTALL.md @@ -1 +1 @@ -Easy ... To Oracle - SQL Server and Sybase ========================================== #### Executable name: EasyTTOra #### Used library: FreeTDS #### Minimum library version: 0.62 #### Recommended library version: 0.91 Preparations ------------ The current stable version of FreeTDS (0.91) does not come as a standard package with all Linux distributions any more. The older version can be used but has some limitiations (maximum varchar length of 255 bytes for example). If you are using the 'standard' version, make sure to also install the development package. On GitHub there is a master version (currently labeled as 0.92) that seems stable but doesn't bring any new features/fixes that are useful for our purposes. Red-Hat/Centos/Oracle: You will need the epel repository. sudo yum install freetds freetds-devel On Ubuntu you can install with: sudo apt-get install freetds-common freetds-bin freetds-dev To install FreeTDS from source: Download the current stable version from freetds.org Unpack it somewhere. Open a shell. Cd into the unpack directory. Enter these commands: ./configure --with-tdsver=7.1 make sudo make install Many more options for the configure command are available, but this works fine. Use flag --with-tdsver=8.0 for FreeTDS releases before 0.91 Windows: The source for FreeTDS 0.91 is included in the ETO download. No further action required. Building EasyTTO ---------------- See the general installation instructions. Specific usage information -------------------------- None Test script ----------- DECLARE r number; BEGIN DBMS_OUTPUT.enable(20000); r := SYS.UTL_ETO.ETO_Create_Database_Connection('MSTEST'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_executable, 'EasyTTOra'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_bin_dir, 'user_dir_eto_bin'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_data_dir, 'user_dir_eto_data'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_name, 'ipadress_or_servername:port'); -- TDS uses 1433 as default tcp port r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_user, 'user'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_pwd, 'password'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_default_charset, 'WE8ISO8859P15'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_def_cacherows, '1'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_list_table_query, 'SELECT ''databasename.dbo.'' + name + '''' as name FROM databasename.sys.all_objects where type in (''U'', ''V'') and is_ms_shipped = 0'); -- Replace databasename with the actual name !!! -- r := SYS.UTL_ETO.ETO_Create_Tables_From_DB ( pOutputSchema => 'SCHEMA' -- replace , pOutputPrefix => 'MS_' , pConnectionname => 'MSTEST' , pDropAllowed => 'Y' ); DBMS_OUTPUT.PUT_LINE('Return value = ' || r); END; ------------------------------------- Copyright (C) 2011-2014 Ivo Herweijer \ No newline at end of file +Easy ... To Oracle - SQL Server and Sybase ========================================== #### Executable name: EasyTTOra #### Used library: FreeTDS #### Minimum library version: 0.62 #### Recommended library version: 0.91 Preparations ------------ The current stable version of FreeTDS (0.91) does not come as a standard package with all Linux distributions any more. The older version can be used but has some limitiations (maximum varchar length of 255 bytes for example). If you are using the 'standard' version, make sure to also install the development package. On GitHub there is a master version (currently labeled as 0.92) that seems stable but doesn't bring any new features/fixes that are useful for our purposes. Red-Hat/Centos/Oracle: You will need the epel repository (https://fedoraproject.org/wiki/EPEL). sudo yum install freetds freetds-devel On Ubuntu you can install with: sudo apt-get install freetds-common freetds-bin freetds-dev To install FreeTDS from source: Download the current stable version from freetds.org Unpack it somewhere. Open a shell. Cd into the unpack directory. Enter these commands: ./configure --with-tdsver=7.1 make sudo make install Many more options for the configure command are available, but this works fine. Use flag --with-tdsver=8.0 for FreeTDS releases before 0.91 Windows: The source for FreeTDS 0.91 is included in the ETO download. No further action required. Building EasyTTO ---------------- See the general installation instructions. Specific usage information -------------------------- None Test script ----------- DECLARE r number; BEGIN DBMS_OUTPUT.enable(20000); r := SYS.UTL_ETO.ETO_Create_Database_Connection('MSTEST'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_executable, 'EasyTTOra'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_bin_dir, 'user_dir_eto_bin'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_data_dir, 'user_dir_eto_data'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_name, 'ipadress_or_servername:port'); -- TDS uses 1433 as default tcp port r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_user, 'user'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_db_pwd, 'password'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_default_charset, 'WE8ISO8859P15'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_def_cacherows, '1'); r := SYS.UTL_ETO.ETO_Set_Parameter('MSTEST', SYS.UTL_ETO.Eto_par_list_table_query, 'SELECT ''databasename.dbo.'' + name + '''' as name FROM databasename.sys.all_objects where type in (''U'', ''V'') and is_ms_shipped = 0'); -- Replace databasename with the actual name !!! -- r := SYS.UTL_ETO.ETO_Create_Tables_From_DB ( pOutputSchema => 'SCHEMA' -- replace , pOutputPrefix => 'MS_' , pConnectionname => 'MSTEST' , pDropAllowed => 'Y' ); DBMS_OUTPUT.PUT_LINE('Return value = ' || r); END; ------------------------------------- Copyright (C) 2011-2014 Ivo Herweijer \ No newline at end of file diff --git a/INSTALL.md b/INSTALL.md index fcda7e0..58e9f52 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1 +1 @@ -Easy-To-Oracle - General ======================== Preparations ------------ See the INSTALL.md files for each component of ETO for installation of required packages. Additionally you will need cmake, version 2.6 or newer. Red-Hat/Centos/Oracle 5&6: sudo yum install cmake Ubuntu: sudo apt-get install cmake Windows: The Windows version does not use cmake. A Visual Studio 2010 solution file is included in the download. The VS solution is set up for 32bit executables. Build ETO --------- Download and unpack ETO somewhere. Open a shell and cd into the easy2oracle-src-X.X.X directory. Enter this command: cmake . If all dependencies could be resolved cmake will have created a Makefile for each ETO component. Enter this command to build ETO: make Or type 'make help' to see all available options. Install ETO ----------- Copy the files from the bin directory to a location that is suitable for your Oracle database server. See file PLSQL/INSTALL.md for detailed instructions. ------------------------------------- Copyright (C) 2011-2014 Ivo Herweijer \ No newline at end of file +Easy-To-Oracle - General ======================== Binary release -------------- Binary releases (i.e. precompiled executables) are available via: [easydatawarehousing.com](http://www.easydatawarehousing.com/easy-to-oracle-download/) Supported platforms are: RHEL, Ubuntu and Windows. Preparations ------------ See the INSTALL.md files for each component of ETO for installation of required packages. Additionally you will need cmake, version 2.6 or newer. Red-Hat/Centos/Oracle 5&6: sudo yum install cmake Ubuntu: sudo apt-get install cmake Windows: The Windows version does not use cmake. A Visual Studio 2010 solution file is included in the download. The VS solution is set up for 32bit executables. Build ETO --------- Download and unpack ETO somewhere. Open a shell and cd into the easy2oracle-src-X.X.X directory. Enter this command: cmake . If all dependencies could be resolved cmake will have created a Makefile for each ETO component. Enter this command to build ETO: make Or type 'make help' to see all available options. Install ETO ----------- Copy the files from the bin directory to a location that is suitable for your Oracle database server. See file PLSQL/INSTALL.md for detailed instructions. ------------------------------------- Copyright (C) 2011-2014 Ivo Herweijer \ No newline at end of file diff --git a/PLSQL/install_utl_eto.sql b/PLSQL/install_utl_eto.sql index 3e6eeda..e4ecc46 100644 --- a/PLSQL/install_utl_eto.sql +++ b/PLSQL/install_utl_eto.sql @@ -48,10 +48,11 @@ AS * * VERSION DATE AUTHOR(S) DESCRIPTION * ------- ----------- --------------- ------------------------------------ - * 1.0.0 15-05-2011 Ivo Herweijer 1st release version - * 1.1.0 15-06-2012 Ivo Herweijer 2nd release. Added MS-Access support - * 2.0.0 15-02-2014 Ivo Herweijer 3rd release. Added Presto support - * 2.0.1 15-05-2014 Ivo Herweijer Add Presto v0.66 types: date/time(stamp) + * 1.0.0 2011-05-15 Ivo Herweijer 1st release version + * 1.1.0 2012-06-15 Ivo Herweijer 2nd release. Added MS-Access support + * 2.0.0 2014-02-15 Ivo Herweijer 3rd release. Added Presto support + * 2.0.1 2014-05-15 Ivo Herweijer Add Presto v0.66 types: date/time(stamp) + * 2.1.0 2014-07-25 Ivo Herweijer Add Oracle support ****************************************************************************/ -- Constants for Describe table -- @@ -80,7 +81,7 @@ Eto_par_enable_logfiles CONSTANT BINARY_INTEGER := 1010; -- Enable logfil Eto_par_enable_badfiles CONSTANT BINARY_INTEGER := 1011; -- Enable badfile: Y = enable, all other values = disable Eto_par_do_not_quote_names CONSTANT BINARY_INTEGER := 1012; -- Skip quoting of the source tablename if it containes spaces. Y = skip quotes, all other values = quote Eto_par_default_charset CONSTANT BINARY_INTEGER := 1013; -- Name of default characterset. Do not set this parameter to use the database default. Can be overwritten when creating tables - -- For MS-SQL server and SQLite 'WE8ISO8859P15' works well, for spreadsheets and Presto use 'AL32UTF8', + -- For MS-SQL server and SQLite 'WE8ISO8859P15' works well, for spreadsheets, Presto and Oracle use 'AL32UTF8', -- for MS-Access use 'WE8ISO8859P1' or 'WE8MSWIN1252' Eto_par_readsize CONSTANT BINARY_INTEGER := 1014; -- Optional: set the value of the external table READSIZE parameter. This is necessary on Windows platform @@ -92,6 +93,7 @@ Eto_bin_easyoto CONSTANT VARCHAR2(30) := 'EasyOTOra'; Eto_bin_easysto CONSTANT VARCHAR2(30) := 'EasySTOra'; Eto_bin_easyato CONSTANT VARCHAR2(30) := 'EasyATOra'; Eto_bin_easypto CONSTANT VARCHAR2(30) := 'EasyPTOra'; +Eto_bin_easynto CONSTANT VARCHAR2(30) := 'EasyNTOra'; -- Oracle2Oracle -- Constants for type conversions - Tabular Data Protocol -- Eto_TDP_ILLEGAL CONSTANT BINARY_INTEGER := 9999; @@ -205,6 +207,19 @@ Eto_PRESTO_TIMESTAMP_TZ CONSTANT BINARY_INTEGER := 60009; Eto_PRESTO_INTERVAL_YM CONSTANT BINARY_INTEGER := 60010; Eto_PRESTO_INTERVAL_DS CONSTANT BINARY_INTEGER := 60011; +-- Constants for type conversions - Oracle -- +Eto_ORACLE_RAW CONSTANT BINARY_INTEGER := 70095; +Eto_ORACLE_NUMBER CONSTANT BINARY_INTEGER := 70002; +Eto_ORACLE_VARCHAR2 CONSTANT BINARY_INTEGER := 70009; +Eto_ORACLE_DATE CONSTANT BINARY_INTEGER := 70012; +Eto_ORACLE_TIME CONSTANT BINARY_INTEGER := 70185; +Eto_ORACLE_TIME_TZ CONSTANT BINARY_INTEGER := 70186; +Eto_ORACLE_TIMESTAMP CONSTANT BINARY_INTEGER := 70187; +Eto_ORACLE_TIMESTAMP_TZ CONSTANT BINARY_INTEGER := 70188; +Eto_ORACLE_TIMESTAMP_LTZ CONSTANT BINARY_INTEGER := 70232; +Eto_ORACLE_INTERVAL_YM CONSTANT BINARY_INTEGER := 70189; +Eto_ORACLE_INTERVAL_DS CONSTANT BINARY_INTEGER := 70190; + -- Functions -- /* Create a new connection. Parameters are stored under the name of this connection @@ -583,7 +598,7 @@ BEGIN END IF; -- Check value for Eto_par_executable. This enables us to use this name to determine which remote platform we are dealing with -- - IF pName = Eto_par_executable AND substr(pValue1, 1, 9) not in (Eto_bin_easytto, Eto_bin_easymto, Eto_bin_easyoto, Eto_bin_easysto, Eto_bin_easyato, Eto_bin_easypto) THEN + IF pName = Eto_par_executable AND substr(pValue1, 1, 9) not in (Eto_bin_easytto, Eto_bin_easymto, Eto_bin_easyoto, Eto_bin_easysto, Eto_bin_easyato, Eto_bin_easypto, Eto_bin_easynto) THEN Raise eValInvalid1; END IF; @@ -954,7 +969,7 @@ BEGIN -- Are there any conversions necessary ? -- XLS types can not be rewritten (since sql is the worksheetname -- FOR i IN gDesc.FIRST .. gDesc.LAST LOOP - IF gDesc(i).T_Datatype in (Eto_TDP_DATETIME, Eto_TDP_DATETIME4, Eto_TDP_BINARY, Eto_TDP_VARBINARY, Eto_TDP_LONGBINARY, Eto_TDP_IMAGE, Eto_TDP_BLOB, Eto_MYSQL_TINY_BLOB, Eto_MYSQL_MEDIUM_BLOB, Eto_MYSQL_LONG_BLOB, Eto_MYSQL_BLOB, Eto_SQLITE_BLOB) THEN + IF gDesc(i).T_Datatype in (Eto_TDP_DATETIME, Eto_TDP_DATETIME4, Eto_TDP_BINARY, Eto_TDP_VARBINARY, Eto_TDP_LONGBINARY, Eto_TDP_IMAGE, Eto_TDP_BLOB, Eto_MYSQL_TINY_BLOB, Eto_MYSQL_MEDIUM_BLOB, Eto_MYSQL_LONG_BLOB, Eto_MYSQL_BLOB, Eto_SQLITE_BLOB, Eto_ORACLE_RAW) THEN vRewrite := true; exit; END IF; @@ -967,7 +982,7 @@ BEGIN FOR i IN gDesc.FIRST .. gDesc.LAST LOOP IF gDesc(i).T_Datatype in (Eto_TDP_DATETIME, Eto_TDP_DATETIME4) THEN vSQL := vSQL || 'Cast(' || gDesc(i).T_Name || ' as datetime2(0)) as ' || gDesc(i).T_Name || ',' || chr(10); - ELSIF gDesc(i).T_Datatype in (Eto_TDP_BINARY, Eto_TDP_VARBINARY, Eto_TDP_LONGBINARY, Eto_TDP_IMAGE, Eto_TDP_BLOB, Eto_MYSQL_TINY_BLOB, Eto_MYSQL_MEDIUM_BLOB, Eto_MYSQL_LONG_BLOB, Eto_MYSQL_BLOB, Eto_SQLITE_BLOB) THEN + ELSIF gDesc(i).T_Datatype in (Eto_TDP_BINARY, Eto_TDP_VARBINARY, Eto_TDP_LONGBINARY, Eto_TDP_IMAGE, Eto_TDP_BLOB, Eto_MYSQL_TINY_BLOB, Eto_MYSQL_MEDIUM_BLOB, Eto_MYSQL_LONG_BLOB, Eto_MYSQL_BLOB, Eto_SQLITE_BLOB, Eto_ORACLE_RAW) THEN gDesc.Delete(i); -- Remove binary fields from nested table -- ELSE vSQL := vSQL || gDesc(i).T_Name || ',' || chr(10); @@ -1193,6 +1208,18 @@ BEGIN WHEN Eto_PRESTO_INTERVAL_YM THEN vOutType := 'interval year to month'; WHEN Eto_PRESTO_INTERVAL_DS THEN vOutType := 'interval day to second'; + WHEN Eto_ORACLE_RAW THEN vOutType := 'varchar2(' || vCharLength || ')'; + WHEN Eto_ORACLE_NUMBER THEN vOutType := 'number'; + WHEN Eto_ORACLE_VARCHAR2 THEN vOutType := 'varchar2(' || vCharLength || ')'; + WHEN Eto_ORACLE_DATE THEN vOutType := 'date'; + WHEN Eto_ORACLE_TIME THEN vOutType := 'timestamp'; + WHEN Eto_ORACLE_TIME_TZ THEN vOutType := 'timestamp with time zone'; + WHEN Eto_ORACLE_TIMESTAMP THEN vOutType := 'timestamp'; + WHEN Eto_ORACLE_TIMESTAMP_TZ THEN vOutType:= 'timestamp with time zone'; + WHEN Eto_ORACLE_TIMESTAMP_LTZ THEN vOutType:='timestamp with local time zone'; + WHEN Eto_ORACLE_INTERVAL_YM THEN vOutType := 'interval year to month'; + WHEN Eto_ORACLE_INTERVAL_DS THEN vOutType := 'interval day to second'; + ELSE vOutType := 'varchar2(4000)'; END CASE; @@ -1344,6 +1371,18 @@ BEGIN WHEN Eto_PRESTO_INTERVAL_YM THEN vOutType := 'char(100) date_format INTERVAL YEAR TO MONTH'; WHEN Eto_PRESTO_INTERVAL_DS THEN vOutType := 'char(100) date_format INTERVAL DAY TO SECOND'; + WHEN Eto_ORACLE_RAW THEN vOutType := 'char(2000)'; + WHEN Eto_ORACLE_NUMBER THEN vOutType := 'unsigned integer external(20)'; + WHEN Eto_ORACLE_VARCHAR2 THEN vOutType := 'char(' || vCharLength || ')'; + WHEN Eto_ORACLE_DATE THEN vOutType := 'char(100) date_format DATE mask "yyyy-mm-dd hh24:mi:ss"'; + WHEN Eto_ORACLE_TIME THEN vOutType := 'char(100) date_format TIMESTAMP mask "hh24:mi:ss.ff6"'; + WHEN Eto_ORACLE_TIME_TZ THEN vOutType := 'char(100) date_format TIMESTAMP WITH TIME ZONE mask "hh24:mi:ss.ff6 tzr"'; + WHEN Eto_ORACLE_TIMESTAMP THEN vOutType := 'char(100) date_format TIMESTAMP mask "yyyy-mm-dd hh24:mi:ss.ff6"'; + WHEN Eto_ORACLE_TIMESTAMP_TZ THEN vOutType:= 'char(100) date_format TIMESTAMP WITH TIME ZONE mask "yyyy-mm-dd hh24:mi:ss.ff6 tzr"'; + WHEN Eto_ORACLE_TIMESTAMP_LTZ THEN vOutType:='char(100) date_format TIMESTAMP WITH TIME ZONE mask "yyyy-mm-dd hh24:mi:ss.ff6 tzr"'; + WHEN Eto_ORACLE_INTERVAL_YM THEN vOutType := 'char(100) date_format INTERVAL YEAR TO MONTH'; + WHEN Eto_ORACLE_INTERVAL_DS THEN vOutType := 'char(100) date_format INTERVAL DAY TO SECOND'; + ELSE vOutType := 'char(4000)'; END CASE; END IF; diff --git a/README.md b/README.md index a21f4c1..50486c3 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -Easy to Oracle - Free open source data integration ================================================== Author: Ivo Herweijer | Easy Datawarehouse Solutions Homepage: http://www.easydatawarehousing.com/easy-to-oracle-free-open-source-data-integration-for-oracle-databases/ What is Easy-To-Oracle ? ------------------------ Easy-To-Oracle is a data integration tool. It can pull data from other databases like Microsoft SQL Server, MySQL, Sybase, SQLite, Presto (Hadoop) and Excel directly into your Oracle 10g/11g/12c database. Easy-To-Oracle (ETO) uses the native communication protocol of the source database, no odbc or jdbc. At the Oracle side the fast and flexible 'ORACLE_LOADER Access Driver' is used. ETO only adds a thin layer to combine the two technologies. As a result ETO is fast ! Benefits -------- - ETO is fast, depending on the speed of the source database 0.5 Gb per minute should easily be attainable - Database centric: Tables on remote databases can be used like any Oracle table. No need for adding database link syntax or quoted names - All functions can be used from within the database using SQL or PL/SQL - Simple to install and use - Can be used for any kind of data-integration, like loading of a datawarehouse, data migration and even creating a data-virtualization platform License ------- This file is part of Easy to Oracle - Free Open Source Data Integration Copyright (C) 2011-2014 Ivo Herweijer Easy to Oracle is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . You can contact me via email: info@easydatawarehousing.com \ No newline at end of file +Easy to Oracle - Free open source data integration ================================================== Author: Ivo Herweijer | Easy Datawarehouse Solutions Homepage: http://www.easydatawarehousing.com/easy-to-oracle-free-open-source-data-integration-for-oracle-databases/ What is Easy-To-Oracle ? ------------------------ Easy-To-Oracle is a data integration tool. It can pull data from other databases like Microsoft SQL Server, MySQL, Sybase, SQLite, Presto (Hadoop) and Excel directly into your Oracle 10g/11g/12c database. Easy-To-Oracle (ETO) uses the native communication protocol of the source database, no odbc or jdbc. At the Oracle side the fast and flexible 'ORACLE_LOADER Access Driver' is used. ETO only adds a thin layer to combine the two technologies. As a result ETO is fast ! Benefits -------- - ETO is fast, depending on the speed of the source database 0.5 Gb per minute should easily be attainable - Database centric: Tables on remote databases can be used like any Oracle table. No need for adding database link syntax or quoted names - All functions can be used from within the database using SQL or PL/SQL - Simple to install and use - Can be used for any kind of data-integration, like loading of a datawarehouse, data migration and even creating a data-virtualization platform - Precompiled binaries available License ------- This file is part of Easy to Oracle - Free Open Source Data Integration Copyright (C) 2011-2014 Ivo Herweijer Easy to Oracle is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . You can contact me via email: info@easydatawarehousing.com \ No newline at end of file diff --git a/VERSION b/VERSION index 49de4e9..e564462 100644 --- a/VERSION +++ b/VERSION @@ -1,6 +1,12 @@ Easy to Oracle - Free open source data integration ================================================== +Version 2.1.0 July 25, 2014 +--------------------------- ++ Added support for Oracle source databases. See EasyNTOra/README.md for the + logic behind this addition. ++ Updated SQLite from version 3.8.3.1 to 3.8.5 + Version 2.0.1 May 15, 2014 -------------------------- + Added support for Presto V0.66 datatypes (date/time/timestamp/interval). Note diff --git a/msvc/Easy-To-Oracle.sln b/msvc/Easy-To-Oracle.sln index 25e0ece..5096a96 100644 --- a/msvc/Easy-To-Oracle.sln +++ b/msvc/Easy-To-Oracle.sln @@ -9,28 +9,56 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EasySTOra", "EasySTOra\Easy EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EasyTTOra", "EasyTTOra\EasyTTOra.vcxproj", "{B91171E9-6F71-403B-B138-A090B0936F89}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EasyNTOra", "EasyNTOra\EasyNTOra.vcxproj", "{4377C4AA-63C4-4FBC-910E-87FD9BDF3796}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 + Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {6C89AD52-8003-4466-9EF9-D40C927A660B}.Debug|Win32.ActiveCfg = Debug|Win32 {6C89AD52-8003-4466-9EF9-D40C927A660B}.Debug|Win32.Build.0 = Debug|Win32 + {6C89AD52-8003-4466-9EF9-D40C927A660B}.Debug|x64.ActiveCfg = Debug|x64 + {6C89AD52-8003-4466-9EF9-D40C927A660B}.Debug|x64.Build.0 = Debug|x64 {6C89AD52-8003-4466-9EF9-D40C927A660B}.Release|Win32.ActiveCfg = Release|Win32 {6C89AD52-8003-4466-9EF9-D40C927A660B}.Release|Win32.Build.0 = Release|Win32 + {6C89AD52-8003-4466-9EF9-D40C927A660B}.Release|x64.ActiveCfg = Release|x64 + {6C89AD52-8003-4466-9EF9-D40C927A660B}.Release|x64.Build.0 = Release|x64 {232CA226-328A-4650-A0F0-A0BACAD8680A}.Debug|Win32.ActiveCfg = Debug|Win32 {232CA226-328A-4650-A0F0-A0BACAD8680A}.Debug|Win32.Build.0 = Debug|Win32 + {232CA226-328A-4650-A0F0-A0BACAD8680A}.Debug|x64.ActiveCfg = Debug|x64 + {232CA226-328A-4650-A0F0-A0BACAD8680A}.Debug|x64.Build.0 = Debug|x64 {232CA226-328A-4650-A0F0-A0BACAD8680A}.Release|Win32.ActiveCfg = Release|Win32 {232CA226-328A-4650-A0F0-A0BACAD8680A}.Release|Win32.Build.0 = Release|Win32 + {232CA226-328A-4650-A0F0-A0BACAD8680A}.Release|x64.ActiveCfg = Release|x64 + {232CA226-328A-4650-A0F0-A0BACAD8680A}.Release|x64.Build.0 = Release|x64 {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Debug|Win32.ActiveCfg = Debug|Win32 {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Debug|Win32.Build.0 = Debug|Win32 + {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Debug|x64.ActiveCfg = Debug|x64 + {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Debug|x64.Build.0 = Debug|x64 {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Release|Win32.ActiveCfg = Release|Win32 {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Release|Win32.Build.0 = Release|Win32 + {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Release|x64.ActiveCfg = Release|x64 + {160D489F-BE96-4EA3-9949-D73CF7E7172B}.Release|x64.Build.0 = Release|x64 {B91171E9-6F71-403B-B138-A090B0936F89}.Debug|Win32.ActiveCfg = Debug|Win32 {B91171E9-6F71-403B-B138-A090B0936F89}.Debug|Win32.Build.0 = Debug|Win32 + {B91171E9-6F71-403B-B138-A090B0936F89}.Debug|x64.ActiveCfg = Debug|x64 + {B91171E9-6F71-403B-B138-A090B0936F89}.Debug|x64.Build.0 = Debug|x64 {B91171E9-6F71-403B-B138-A090B0936F89}.Release|Win32.ActiveCfg = Release|Win32 {B91171E9-6F71-403B-B138-A090B0936F89}.Release|Win32.Build.0 = Release|Win32 + {B91171E9-6F71-403B-B138-A090B0936F89}.Release|x64.ActiveCfg = Release|x64 + {B91171E9-6F71-403B-B138-A090B0936F89}.Release|x64.Build.0 = Release|x64 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Debug|Win32.ActiveCfg = Debug|Win32 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Debug|Win32.Build.0 = Debug|Win32 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Debug|x64.ActiveCfg = Debug|x64 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Debug|x64.Build.0 = Debug|x64 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Release|Win32.ActiveCfg = Release|Win32 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Release|Win32.Build.0 = Release|Win32 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Release|x64.ActiveCfg = Release|x64 + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/msvc/EasyNTOra/Debug/This is where dll and sym files go b/msvc/EasyNTOra/Debug/This is where dll and sym files go new file mode 100644 index 0000000..e69de29 diff --git a/msvc/EasyNTOra/EasyNTOra.vcxproj b/msvc/EasyNTOra/EasyNTOra.vcxproj new file mode 100644 index 0000000..cca6c71 --- /dev/null +++ b/msvc/EasyNTOra/EasyNTOra.vcxproj @@ -0,0 +1,154 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {4377C4AA-63C4-4FBC-910E-87FD9BDF3796} + EasyNTOra + + + + Application + true + MultiByte + + + Application + true + MultiByte + + + Application + false + true + MultiByte + + + Application + false + true + MultiByte + + + + + + + + + + + + + + + + + + + $(SolutionDir)$(ProjectName)\$(Configuration)\ + + + $(SolutionDir)$(ProjectName)\$(Configuration)\ + + + $(SolutionDir)$(ProjectName)\$(Configuration)\ + + + $(SolutionDir)$(ProjectName)\$(Configuration)\ + + + + Level3 + Disabled + ..\..\EasyNTOra\include;..\..\ETO_Common\include;..\..\EasyNTOra\instantclient\include + 4996 + + + true + ..\..\EasyNTOra\instantclient\lib + oci.lib;%(AdditionalDependencies) + + + + + Level3 + Disabled + ..\..\EasyNTOra\include;..\..\ETO_Common\include;..\..\EasyNTOra\instantclient\include + 4996 + + + true + ..\..\EasyNTOra\instantclient\lib + oci.lib;%(AdditionalDependencies) + + + + + Level3 + MaxSpeed + true + true + ..\..\EasyNTOra\include;..\..\ETO_Common\include;..\..\EasyNTOra\instantclient\include + 4996 + + + true + true + true + oci.lib;%(AdditionalDependencies) + ..\..\EasyNTOra\instantclient\lib + + + + + Level3 + MaxSpeed + true + true + ..\..\EasyNTOra\include;..\..\ETO_Common\include + 4996 + + + true + true + true + ..\..\EasyNTOra\instantclient\lib + oci.lib;%(AdditionalDependencies) + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msvc/EasyNTOra/EasyNTOra.vcxproj.filters b/msvc/EasyNTOra/EasyNTOra.vcxproj.filters new file mode 100644 index 0000000..cb29d7e --- /dev/null +++ b/msvc/EasyNTOra/EasyNTOra.vcxproj.filters @@ -0,0 +1,63 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {9b6bfaa2-c9ad-497f-a556-d4aef7e11996} + + + {0cd61c7b-5ffe-4d87-99ff-d25e621fd4f2} + + + {bf81ecdd-8b60-4009-b9a0-b44bc503dbd2} + + + + + common\include + + + common\include + + + common\include + + + common\include + + + common\include + + + + + common\src + + + common\src + + + common\src + + + common\src + + + common\src + + + source + + + \ No newline at end of file diff --git a/msvc/EasyNTOra/EasyNTOra.vcxproj.user b/msvc/EasyNTOra/EasyNTOra.vcxproj.user new file mode 100644 index 0000000..d12a447 --- /dev/null +++ b/msvc/EasyNTOra/EasyNTOra.vcxproj.user @@ -0,0 +1,7 @@ + + + + ..\..\EasyNTOra\Oracle_testfile.ini + WindowsLocalDebugger + + \ No newline at end of file diff --git a/msvc/EasyNTOra/Release/This is where dll files go b/msvc/EasyNTOra/Release/This is where dll files go new file mode 100644 index 0000000..e69de29 diff --git a/msvc/EasyPTOra/EasyPTOra.vcxproj b/msvc/EasyPTOra/EasyPTOra.vcxproj index ee74598..89f3b42 100644 --- a/msvc/EasyPTOra/EasyPTOra.vcxproj +++ b/msvc/EasyPTOra/EasyPTOra.vcxproj @@ -67,6 +67,9 @@ $(SolutionDir)$(ProjectName)\$(Configuration)\ + + $(SolutionDir)$(ProjectName)\$(Configuration)\ + Level4