我们了解sqlite3数据库是一个文件格式的数据库,那显然它必然至少会持有一个打开的文件描述符,本文简单介绍 sqlite3结构体本身与这个文件描述符之间的关系,以及调用sqlite3_open最终打开这个文件描述符的流程。
先上结论,在sqlite3的数据结构中,与文件描述符的定义关系大致如下流程所示:
struct sqlite3——>(struct Db)aDb——>(struct Btree *)pBt——>
(struct BtShared *)pBt——>(struct Pager *)pPager ——>
(struct sqlite3_file)fd = struct unixFile ——>(int)h;
for reference
struct sqlite3 line 16840
aDb代表这个数据库句柄上打开的数据库文件,aDb[0]为主数据库,可以包含多个,比如内存数据库,以及attach上来的数据库。
struct Btree .definition in line 66184
sqlite3_file的定义根据VFS的实现不同而不同。在默认的unix中,是如下定义
/*
** The unixFile structure is subclass of sqlite3_file specific to the unix
** VFS implementations.
*/
typedef struct unixFile unixFile;
下面分析一下打开数据库文件的流程
在最基本的使用方式中,我们通过如下的方式打开一个sqlite3数据库的句柄。
sqlite3* db;
int iRet = sqlite3_open("test.db",&db);
整体的调用链如下,中间还包括了其他一些填充的流程。
sqlite3_open——》openDatabase——》sqlite3BtreeOpen——》sqlite3PagerOpen——》
sqlite3OsOpen——》 pVfs->xOpen ——》unixOpen ——》robust_open
在我这里使用的 3.39.4 版本中,其实际调用了openDatabase函数。
/*
** Open a new database handle.
*/
SQLITE_API int sqlite3_open(
const char *zFilename,
sqlite3 **ppDb
){
return openDatabase(zFilename, ppDb,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0);
}
openDatabase函数中执行了数据库handle的分配和初始化,这里注意下openDatabase携带了4个参数,除了需要打开的数据库路径和需要返回出来的sqlite3的Handle,另外两个参数分别是打开的flag和应使用的操作系统接口对应的操作系统名称。这两个参数,在一个不同于sqlite3_open的接口 sqlite3_open_v2接口中是允许配置的,该接口的语义可以参考如下网址。https://www.sqlite.net.cn/c3ref/open.html
按照默认的不配置的老接口,会使用默认的sqlite3_vfs对象。该对象实际是sqlite3结构体的第一个参数,也是我们后面将会用到的很重要的一个成员。
static int openDatabase(
const char *zFilename, /* Database filename UTF-8 encoded */
sqlite3 **ppDb, /* OUT: Returned database handle */
unsigned int flags, /* Operational flags */
const char *zVfs /* Name of the VFS to use */
){
sqlite3 *db; /* Store allocated handle here */
...
rc = sqlite3_initialize();
if( rc ) return rc;
...
db = sqlite3MallocZero( sizeof(sqlite3) );
...
/*关键的路径参数被传递给了 sqlite3ParseUri,同时将给db的pVfs参数,也就是 sqlite3_vfs 对象进行了赋值,下文提供了sqlite3ParseUri的定义*/
rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg);
...
/* Open the backend database driver */
rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0,
flags | SQLITE_OPEN_MAIN_DB);
...
}
sqlite3ParseUri完成了比较关键的解析,他接受了完整的路径入参,默认该路径为uri的形式,然后提取出路径赋值到 zOpen中,并根据zVfs的参数,给VFS赋值。因为这里没有赋值,所以默认返回vfsList中的第一个vfs。这个vfsList在sqlite3_os_init中被初始化和定义。sqlite3_os_init为内部初始化调用,在sqlite3_initialize中被初始化。是openDatabase函数的第一步。
SQLITE_PRIVATE int sqlite3ParseUri(
const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */
const char *zUri, /* Nul-terminated URI to parse */
unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */
sqlite3_vfs **ppVfs, /* OUT: VFS to use */
char **pzFile, /* OUT: Filename component of URI */
char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */
){
}
然后根据上述流程在 openDatabase中,接力棒给到了sqlite3BtreeOpen,该函数执行了实际数据库流程的打开工作,这里比较复杂的流程是需要把打开的数据库实例对应的 BtShared 关联到BtTree,再关联到sqlite3的aDb数组。
真正执行打开的关键操作是在sqlite3PagerOpen。其会返回BtShared持有的pPager,BtShared代表一个数据库文件的实例。
line 69301
/*
** Open a database file.
**...
*/
SQLITE_PRIVATE int sqlite3BtreeOpen(
sqlite3_vfs *pVfs, /* VFS to use for this b-tree */
const char *zFilename, /* Name of the file containing the BTree database */
sqlite3 *db, /* Associated database handle */
Btree **ppBtree, /* Pointer to new Btree object written here */
int flags, /* Options */
int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */
){
...
rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename,
sizeof(MemPage), flags, vfsFlags, pageReinit);
...
sqlite3PagerOpen函数打开了一个Pager对象,并将其赋值给BtShared,根据Pager的定义,每一个打开的文件,都被一个独立的Pager实例结构体管理 。其打开调用了 sqlite3OsOpen,而sqlite3OsOpen 又调用了 pVfs->xOpen ,即我们之前提到的 sqlite3_vfs 中的xOpen的handle。
/*
** Each open file is managed by a separate instance of the "Pager" structure.
*/
typedef struct Pager Pager;
...
line 58590
SQLITE_PRIVATE int sqlite3PagerOpen(
sqlite3_vfs *pVfs, /* The virtual file system to use */
Pager **ppPager, /* OUT: Return the Pager structure here */
const char *zFilename, /* Name of the database file to open */
int nExtra, /* Extra bytes append to each in-memory page */
int flags, /* flags controlling this file */
int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */
void (*xReinit)(DbPage*) /* Function to reinitialize pages */
){
...
rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout);
...
}
...
SQLITE_PRIVATE int sqlite3OsOpen(
sqlite3_vfs *pVfs,
const char *zPath,
sqlite3_file *pFile,
int flags,
int *pFlagsOut
){
int rc;
DO_OS_MALLOC_TEST(0);
/* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed
** down into the VFS layer. Some SQLITE_OPEN_ flags (for example,
** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before
** reaching the VFS. */
rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut);
assert( rc==SQLITE_OK || pFile->pMethods==0 );
return rc;
}
sqlite3_vfs的handle的定义在sqlite3_os_init中。默认open的定义为 unixOpen 函数,unixOpen函数的定义如下
static int unixOpen(
sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */
const char *zPath, /* Pathname of file to be opened */
sqlite3_file *pFile, /* The file descriptor to be filled in */
int flags, /* Input flags to control the opening */
int *pOutFlags /* Output flags returned to SQLite core */
){
...
fd = robust_open(zName, openFlags, openMode);
...
...
rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags);
...
}
robust_open实际执行了打开的动作,而fillInUnixFile 则是将fd填充到了对应 的pFile句柄里。
static int robust_open(const char *z, int f, mode_t m){
int fd;
mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS;
while(1){
#if defined(O_CLOEXEC)
fd = osOpen(z,f|O_CLOEXEC,m2);
#else
fd = osOpen(z,f,m2);
#endif
if( fd<0 ){ if( errno="=EINTR" ) continue; break; } fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break;
osClose(fd);
sqlite3_log(SQLITE_WARNING,
"attempt to open \"%s\" as file descriptor %d", z, fd);
fd = -1;
if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; } if( fd>=0 ){
if( m!=0 ){
struct stat statbuf;
if( osFstat(fd, &statbuf)==0
&& statbuf.st_size==0
&& (statbuf.st_mode&0777)!=m
){
osFchmod(fd, m);
}
}
#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0)
osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC);
#endif
}
return fd;
}
</0></0>
作为参考,附上sqlite3的结构体定义如下
struct sqlite3 {
sqlite3_vfs *pVfs; /* OS Interface */
struct Vdbe *pVdbe; /* List of active virtual machines */
CollSeq *pDfltColl; /* BINARY collseq for the database encoding */
sqlite3_mutex *mutex; /* Connection mutex */
Db *aDb; /* All backends */
int nDb; /* Number of backends currently in use */
u32 mDbFlags; /* flags recording internal state */
u64 flags; /* flags settable by pragmas. See below */
i64 lastRowid; /* ROWID of most recent insert (see above) */
i64 szMmap; /* Default mmap_size setting */
u32 nSchemaLock; /* Do not reset the schema when non-zero */
unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */
int errCode; /* Most recent error code (SQLITE_*) */
int errByteOffset; /* Byte offset of error in SQL statement */
int errMask; /* & result codes with this before returning */
int iSysErrno; /* Errno value from last system error */
u32 dbOptFlags; /* Flags to enable/disable optimizations */
u8 enc; /* Text encoding */
u8 autoCommit; /* The auto-commit flag. */
u8 temp_store; /* 1: file 2: memory 0: default */
u8 mallocFailed; /* True if we have seen a malloc failure */
u8 bBenignMalloc; /* Do not require OOMs if true */
u8 dfltLockMode; /* Default locking-mode for attached dbs */
signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */
u8 suppressErr; /* Do not issue error messages if true */
u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */
u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */
u8 mTrace; /* zero or more SQLITE_TRACE flags */
u8 noSharedCache; /* True if no shared-cache backends */
u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */
u8 eOpenState; /* Current condition of the connection */
int nextPagesize; /* Pagesize after VACUUM if >0 */
i64 nChange; /* Value returned by sqlite3_changes() */
i64 nTotalChange; /* Value returned by sqlite3_total_changes() */
int aLimit[SQLITE_N_LIMIT]; /* Limits */
int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */
struct sqlite3InitInfo { /* Information used during initialization */
Pgno newTnum; /* Rootpage of table being initialized */
u8 iDb; /* Which db file is being initialized */
u8 busy; /* TRUE if currently initializing */
unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */
unsigned imposterTable : 1; /* Building an imposter table */
unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */
const char **azInit; /* "type", "name", and "tbl_name" columns */
} init;
int nVdbeActive; /* Number of VDBEs currently running */
int nVdbeRead; /* Number of active VDBEs that read or write */
int nVdbeWrite; /* Number of active VDBEs that read and write */
int nVdbeExec; /* Number of nested calls to VdbeExec() */
int nVDestroy; /* Number of active OP_VDestroy operations */
int nExtension; /* Number of loaded extensions */
void **aExtension; /* Array of shared library handles */
union {
void (*xLegacy)(void*,const char*); /* mTrace==SQLITE_TRACE_LEGACY */
int (*xV2)(u32,void*,void*,void*); /* All other mTrace values */
} trace;
void *pTraceArg; /* Argument to the trace function */
#ifndef SQLITE_OMIT_DEPRECATED
void (*xProfile)(void*,const char*,u64); /* Profiling function */
void *pProfileArg; /* Argument to profile function */
#endif
void *pCommitArg; /* Argument to xCommitCallback() */
int (*xCommitCallback)(void*); /* Invoked at every commit. */
void *pRollbackArg; /* Argument to xRollbackCallback() */
void (*xRollbackCallback)(void*); /* Invoked at every commit. */
void *pUpdateArg;
void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
void *pAutovacPagesArg; /* Client argument to autovac_pages */
void (*xAutovacDestr)(void*); /* Destructor for pAutovacPAgesArg */
unsigned int (*xAutovacPages)(void*,const char*,u32,u32,u32);
Parse *pParse; /* Current parse */
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
void *pPreUpdateArg; /* First argument to xPreUpdateCallback */
void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */
void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64
);
PreUpdate *pPreUpdate; /* Context for active pre-update callback */
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
#ifndef SQLITE_OMIT_WAL
int (*xWalCallback)(void *, sqlite3 *, const char *, int);
void *pWalArg;
#endif
void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*);
void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*);
void *pCollNeededArg;
sqlite3_value *pErr; /* Most recent error message */
union {
volatile int isInterrupted; /* True if sqlite3_interrupt has been called */
double notUsed1; /* Spacer */
} u1;
Lookaside lookaside; /* Lookaside malloc configuration */
#ifndef SQLITE_OMIT_AUTHORIZATION
sqlite3_xauth xAuth; /* Access authorization function */
void *pAuthArg; /* 1st argument to the access auth function */
#endif
#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
int (*xProgress)(void *); /* The progress callback */
void *pProgressArg; /* Argument to the progress callback */
unsigned nProgressOps; /* Number of opcodes for progress callback */
#endif
#ifndef SQLITE_OMIT_VIRTUALTABLE
int nVTrans; /* Allocated size of aVTrans */
Hash aModule; /* populated by sqlite3_create_module() */
VtabCtx *pVtabCtx; /* Context for active vtab connect/create */
VTable **aVTrans; /* Virtual tables with open transactions */
VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */
#endif
Hash aFunc; /* Hash table of connection functions */
Hash aCollSeq; /* All collating sequences */
BusyHandler busyHandler; /* Busy callback */
Db aDbStatic[2]; /* Static space for the 2 default backends */
Savepoint *pSavepoint; /* List of active savepoints */
int nAnalysisLimit; /* Number of index rows to ANALYZE */
int busyTimeout; /* Busy handler timeout, in msec */
int nSavepoint; /* Number of non-transaction savepoints */
int nStatement; /* Number of nested statement-transactions */
i64 nDeferredCons; /* Net deferred constraints this transaction. */
i64 nDeferredImmCons; /* Net deferred immediate constraints */
int *pnBytesFreed; /* If not NULL, increment this in DbFree() */
#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
/* The following variables are all protected by the STATIC_MAIN
** mutex, not by sqlite3.mutex. They are used by code in notify.c.
**
** When X.pUnlockConnection==Y, that means that X is waiting for Y to
** unlock so that it can proceed.
**
** When X.pBlockingConnection==Y, that means that something that X tried
** tried to do recently failed with an SQLITE_LOCKED error due to locks
** held by Y.
*/
sqlite3 *pBlockingConnection; /* Connection that caused SQLITE_LOCKED */
sqlite3 *pUnlockConnection; /* Connection to watch for unlock */
void *pUnlockArg; /* Argument to xUnlockNotify */
void (*xUnlockNotify)(void **, int); /* Unlock notify callback */
sqlite3 *pNextBlocked; /* Next in list of all blocked connections */
#endif
#ifdef SQLITE_USER_AUTHENTICATION
sqlite3_userauth auth; /* User authentication information */
#endif
};
sqlite3_vfs的定义
struct sqlite3_vfs {
int iVersion; /* Structure version number (currently 3) */
int szOsFile; /* Size of subclassed sqlite3_file */
int mxPathname; /* Maximum file pathname length */
sqlite3_vfs *pNext; /* Next registered VFS */
const char *zName; /* Name of this virtual file system */
void *pAppData; /* Pointer to application-specific data */
int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*,
int flags, int *pOutFlags);
int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir);
int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut);
int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut);
void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename);
void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg);
void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void);
void (*xDlClose)(sqlite3_vfs*, void*);
int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut);
int (*xSleep)(sqlite3_vfs*, int microseconds);
int (*xCurrentTime)(sqlite3_vfs*, double*);
int (*xGetLastError)(sqlite3_vfs*, int, char *);
/*
** The methods above are in version 1 of the sqlite_vfs object
** definition. Those that follow are added in version 2 or later
*/
int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*);
/*
** The methods above are in versions 1 and 2 of the sqlite_vfs object.
** Those below are for version 3 and greater.
*/
int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr);
sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName);
const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName);
/*
** The methods above are in versions 1 through 3 of the sqlite_vfs object.
** New fields may be appended in future versions. The iVersion
** value will increment whenever this happens.
*/
};
Btree相关
/* A Btree handle
**
** A database connection contains a pointer to an instance of
** this object for every database file that it has open. This structure
** is opaque to the database connection. The database connection cannot
** see the internals of this structure and only deals with pointers to
** this structure.
**
** For some database files, the same underlying database cache might be
** shared between multiple connections. In that case, each connection
** has it own instance of this object. But each instance of this object
** points to the same BtShared object. The database cache and the
** schema associated with the database file are all contained within
** the BtShared object.
**
** All fields in this structure are accessed under sqlite3.mutex.
** The pBt pointer itself may not be changed while there exists cursors
** in the referenced BtShared that point back to this Btree since those
** cursors have to go through this Btree to find their BtShared and
** they often do so without holding sqlite3.mutex.
*/
struct Btree {
sqlite3 *db; /* The database connection holding this btree */
BtShared *pBt; /* Sharable content of this btree */
u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */
u8 sharable; /* True if we can share pBt with another db */
u8 locked; /* True if db currently has pBt locked */
u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */
int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */
int nBackup; /* Number of backup operations reading this btree */
u32 iBDataVersion; /* Combines with pBt->pPager->iDataVersion */
Btree *pNext; /* List of other sharable Btrees from the same db */
Btree *pPrev; /* Back pointer of the same list */
#ifdef SQLITE_DEBUG
u64 nSeek; /* Calls to sqlite3BtreeMovetoUnpacked() */
#endif
#ifndef SQLITE_OMIT_SHARED_CACHE
BtLock lock; /* Object used to lock page 1 */
#endif
};
/*
** An instance of this object represents a single database file.
**
** A single database file can be in use at the same time by two
** or more database connections. When two or more connections are
** sharing the same database file, each connection has it own
** private Btree object for the file and each of those Btrees points
** to this one BtShared object. BtShared.nRef is the number of
** connections currently sharing this database file.
**
** Fields in this structure are accessed under the BtShared.mutex
** mutex, except for nRef and pNext which are accessed under the
** global SQLITE_MUTEX_STATIC_MAIN mutex. The pPager field
** may not be modified once it is initially set as long as nRef>0.
** The pSchema field may be set once under BtShared.mutex and
** thereafter is unchanged as long as nRef>0.
**
** isPending:
**
** If a BtShared client fails to obtain a write-lock on a database
** table (because there exists one or more read-locks on the table),
** the shared-cache enters 'pending-lock' state and isPending is
** set to true.
**
** The shared-cache leaves the 'pending lock' state when either of
** the following occur:
**
** 1) The current writer (BtShared.pWriter) concludes its transaction, OR
** 2) The number of locks held by other connections drops to zero.
**
** while in the 'pending-lock' state, no connection may start a new
** transaction.
**
** This feature is included to help prevent writer-starvation.
*/
struct BtShared {
Pager *pPager; /* The page cache */
sqlite3 *db; /* Database connection currently using this Btree */
BtCursor *pCursor; /* A list of all open cursors */
MemPage *pPage1; /* First page of the database */
u8 openFlags; /* Flags to sqlite3BtreeOpen() */
#ifndef SQLITE_OMIT_AUTOVACUUM
u8 autoVacuum; /* True if auto-vacuum is enabled */
u8 incrVacuum; /* True if incr-vacuum is enabled */
u8 bDoTruncate; /* True to truncate db on commit */
#endif
u8 inTransaction; /* Transaction state */
u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */
u8 nReserveWanted; /* Desired number of extra bytes per page */
u16 btsFlags; /* Boolean parameters. See BTS_* macros below */
u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */
u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */
u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */
u16 minLeaf; /* Minimum local payload in a LEAFDATA table */
u32 pageSize; /* Total number of bytes on a page */
u32 usableSize; /* Number of usable bytes on each page */
int nTransaction; /* Number of open transactions (read + write) */
u32 nPage; /* Number of pages in the database */
void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */
void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */
sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */
Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */
#ifndef SQLITE_OMIT_SHARED_CACHE
int nRef; /* Number of references to this structure */
BtShared *pNext; /* Next on a list of sharable BtShared structs */
BtLock *pLock; /* List of locks held on this shared-btree struct */
Btree *pWriter; /* Btree with currently open write transaction */
#endif
u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */
int nPreformatSize; /* Size of last cell written by TransferRow() */
};
Original: https://blog.csdn.net/u010050543/article/details/128277309
Author: darkpush
Title: sqlite3结构体与实际打开的文件描述符之间的关系
原创文章受到原创版权保护。转载请注明出处:https://www.johngo689.com/816837/
转载文章受原作者版权保护。转载请注明原作者出处!