From a5698c60bc3a56493636663b43a3540393ae4878 Mon Sep 17 00:00:00 2001 From: peitingwei Date: Wed, 29 Mar 2023 18:47:46 +0800 Subject: [PATCH] hot data precache for index --- src/backend/catalog/pg_hot_data.c | 66 ++++++++++++++++++++--- src/backend/executor/execMain.c | 2 +- src/backend/executor/nodeBitmapHeapscan.c | 3 ++ src/backend/executor/nodeIndexscan.c | 4 ++ src/backend/storage/buffer/bufmgr.c | 24 ++++++++- src/backend/tcop/postgres.c | 22 +++++--- src/include/catalog/pg_hot_data.h | 15 ++++++ src/include/storage/bufmgr.h | 4 +- 8 files changed, 123 insertions(+), 17 deletions(-) diff --git a/src/backend/catalog/pg_hot_data.c b/src/backend/catalog/pg_hot_data.c index bd4ecf0..f1dffdb 100644 --- a/src/backend/catalog/pg_hot_data.c +++ b/src/backend/catalog/pg_hot_data.c @@ -22,7 +22,7 @@ void PrecacheHotData() char primaryUser[NAMEDATALEN]; //default:postgres char primaryPw[NAMEDATALEN]; //default:123456 char primaryPort[8]; //default:PostPortNumber - char localPort[8]; //default:master + char localPort[8]; //default:PostPortNumber StringInfoData cmd, primaryConnStr, localConnStr; initStringInfo(&cmd); @@ -181,7 +181,7 @@ void PrecacheHotData() return; } - appendStringInfoString(&cmd, "SELECT datname, relname, crules FROM pg_hot_data WHERE crulessettime>cachetime AND clientname='"); + appendStringInfoString(&cmd, "SELECT datname, relname, crules, ctype, indexname, keyname, keyvalue, keytype FROM pg_hot_data WHERE crulessettime>cachetime AND clientname='"); appendStringInfoString(&cmd, instanceName); appendStringInfoString(&cmd, "'"); @@ -200,9 +200,19 @@ void PrecacheHotData() char *datname; char *relname; char *crules; - datname = PQgetvalue(ruleRes, i, 0); - relname = PQgetvalue(ruleRes, i, 1); - crules = PQgetvalue(ruleRes, i, 2); + char *ctype; + char *indexname; + char *keyname; + char *keyvalue; + char *keytype; + datname = PQgetvalue(ruleRes, i, 0); + relname = PQgetvalue(ruleRes, i, 1); + crules = PQgetvalue(ruleRes, i, 2); + ctype = PQgetvalue(ruleRes, i, 3); + indexname = PQgetvalue(ruleRes, i, 4); + keyname = PQgetvalue(ruleRes, i, 5); + keyvalue = PQgetvalue(ruleRes, i, 6); + keytype = PQgetvalue(ruleRes, i, 7); //precache hot data(table level) if (strcmp(crules, "t") == 0) @@ -221,8 +231,43 @@ void PrecacheHotData() continue; } resetStringInfo(&cmd); - appendStringInfoString(&cmd, "precache select * from "); - appendStringInfoString(&cmd, relname); + + if (strcmp(ctype, "t") == 0) + { + appendStringInfoString(&cmd, "precache table select * from "); + appendStringInfoString(&cmd, relname); + } + else if (strcmp(ctype, "i") == 0) + { + if (strcmp(keytype, "s") == 0) + { + appendStringInfoString(&cmd, "precache index select * from "); + appendStringInfoString(&cmd, relname); + appendStringInfoString(&cmd, " where "); + appendStringInfoString(&cmd, keyname); + appendStringInfoString(&cmd, "='"); + appendStringInfoString(&cmd, keyvalue); + appendStringInfoString(&cmd, "'"); + } + else if (strcmp(keytype, "n") == 0) + { + appendStringInfoString(&cmd, "precache index select * from "); + appendStringInfoString(&cmd, relname); + appendStringInfoString(&cmd, " where "); + appendStringInfoString(&cmd, keyname); + appendStringInfoString(&cmd, "="); + appendStringInfoString(&cmd, keyvalue); + } + else + { + continue; + } + } + else + { + continue; + } + PGresult *precacheRes = PQexec(precacheConn, cmd.data); if (PQresultStatus(precacheRes) != PGRES_TUPLES_OK) @@ -248,6 +293,13 @@ void PrecacheHotData() appendStringInfoString(&cmd, relname); appendStringInfoString(&cmd, "' AND crules='"); appendStringInfoString(&cmd, crules); + appendStringInfoString(&cmd, "' AND ctype='"); + appendStringInfoString(&cmd, ctype); + if (strcmp(ctype, "i") == 0) + { + appendStringInfoString(&cmd, "' AND indexname='"); + appendStringInfoString(&cmd, indexname); + } appendStringInfoString(&cmd, "' AND clientname='"); appendStringInfoString(&cmd, instanceName); appendStringInfoString(&cmd, "'"); diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index ab85119..8e74118 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1557,7 +1557,7 @@ ExecutePlan(EState *estate, if (TupIsNull(slot)) break; - if (!isPreCache) + if (!isPreCacheTable && !isPreCacheIndex) { /* * If we have a junk filter, then project a new tuple with the junk diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 2db1914..5ecc61f 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -51,6 +51,7 @@ #include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/spccache.h" +#include "storage/bufmgr.h" static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node); static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate); @@ -81,6 +82,8 @@ BitmapHeapNext(BitmapHeapScanState *node) ParallelBitmapHeapState *pstate = node->pstate; dsa_area *dsa = node->ss.ps.state->es_query_dsa; + preCacheIndexNode = ((BitmapIndexScanState *)((PlanState *)(node))->lefttree)->biss_ScanDesc->indexRelation->rd_node->relNode; + /* * extract necessary information from index scan node */ diff --git a/src/backend/executor/nodeIndexscan.c b/src/backend/executor/nodeIndexscan.c index 2fffb1b..e74b7cd 100644 --- a/src/backend/executor/nodeIndexscan.c +++ b/src/backend/executor/nodeIndexscan.c @@ -43,6 +43,7 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" +#include "storage/bufmgr.h" /* * When an ordering operator is used, tuples fetched from the index that @@ -86,6 +87,9 @@ IndexNext(IndexScanState *node) IndexScanDesc scandesc; TupleTableSlot *slot; + /* set preCacheIndexNode */ + preCacheIndexNode = node->iss_RelationDesc->rd_node->relNode; + /* * extract necessary information from index scan node */ diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 79c3773..71529a1 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -791,6 +791,18 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, * miss. */ pgstat_count_buffer_read(reln); + /* precache index */ + if (isPreCacheIndex) + { + BlockNumber precacheblocks; + precacheblocks = smgrnblocks(eln->rd_smgr, forkNum); + for(BlockNumber i=0; i < precacheblocks; i++) + { + ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, forkNum, i, mode, strategy, &hit); + } + isPreCacheIndex = false; + } + buf = ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, forkNum, blockNum, mode, strategy, &hit); if (hit) @@ -920,7 +932,17 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, // for precache: buf not be eliminated by clock algorithm if (needPreCacheEscape) { - bufHdr->isPreCacheEscape=true; + if (isPreCacheIndex) + { + if (preCacheIndexNode == bufHdr->tag->rnode->relNode) + { + bufHdr->isPreCacheEscape=true; + } + } + else if (isPreCacheTable) + { + bufHdr->isPreCacheEscape=true; + } } } diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 3a91478..611b533 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -86,8 +86,10 @@ * global variables * ---------------- */ -bool isPreCache = false; +bool isPreCacheTable = false; +bool isPreCacheIndex = false; bool needPreCacheEscape = false; +Oid preCacheIndexNode = 0; const char *debug_query_string; /* client-supplied query string */ /* Note: whereToSendOutput is initialized for the bootstrap/standalone case */ @@ -1213,7 +1215,7 @@ exec_simple_query(const char *query_string) */ MemoryContextSwitchTo(oldcontext); - if (isPreCache) + if (isPreCacheTable || isPreCacheIndex) { needPreCacheEscape = true; } @@ -1228,7 +1230,7 @@ exec_simple_query(const char *query_string) receiver, &qc); - if (isPreCache) + if (isPreCacheTable || isPreCacheIndex) { needPreCacheEscape = false; } @@ -4504,11 +4506,17 @@ PostgresMain(int argc, char *argv[], bool PrivateConn, } else { - if (strstr(query_string, "precache ") != NULL && query_string - strstr(query_string, "precache ") == 0) + if (strstr(query_string, "precache table ") != NULL && query_string - strstr(query_string, "precache table ") == 0) { - isPreCache = true; - exec_simple_query(query_string + strlen("precache ")); - isPreCache = false; + isPreCacheTable = true; + exec_simple_query(query_string + strlen("precache table ")); + isPreCacheTable = false; + } + else if (strstr(query_string, "precache index ") != NULL && query_string - strstr(query_string, "precache index ") == 0) + { + isPreCacheIndex = true; + exec_simple_query(query_string + strlen("precache index ")); + isPreCacheIndex = false; } else { diff --git a/src/include/catalog/pg_hot_data.h b/src/include/catalog/pg_hot_data.h index 0f22a32..6ed9dd3 100644 --- a/src/include/catalog/pg_hot_data.h +++ b/src/include/catalog/pg_hot_data.h @@ -36,6 +36,21 @@ CATALOG(pg_hot_data,4790,HotDataRelationId) BKI_SHARED_RELATION BKI_ROWTYPE_OID( /* caching rules */ char crules; + /* cache type: 't' for table, 'i' for index */ + char ctype; + + /* index name */ + NameData indexname; + + /* key name */ + NameData keyname; + + /* key value */ + NameData keyvalue; + + /* key type: 's' for string, 'n' for numeric */ + char keytype; + /* client name */ NameData clientname; diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index f2fa539..a942521 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -79,8 +79,10 @@ extern int bgwriter_flush_after; extern bool bulk_io_is_in_progress; extern int bulk_io_in_progress_count; -extern bool isPreCache; +extern bool isPreCacheTable; +extern bool isPreCacheIndex; extern bool needPreCacheEscape; +extern Oid preCacheIndexNode; /* in buf_init.c */ extern PGDLLIMPORT char *BufferBlocks;