test/he3pg
1
0
Fork 0
mirror of https://gitee.com/he3db/he3pg.git synced 2024-12-02 04:07:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'dev_performance' of gitee.com:he3db/he3pg into dev_performance

Browse Source
This commit is contained in:
shenzhengntu 2023-03-31 17:10:27 +08:00
commit c9d2c383b9
14 changed files with 161 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/catalog/catalog.c
View File

@ -265,7 +265,7 @@ IsSharedRelation(Oid relationId)
relationId == AuthMemMemRoleIndexId ||
relationId == DatabaseNameIndexId ||
relationId == DatabaseOidIndexId ||
relationId == HotDataDatnameRelnameIndexId ||
relationId == HotDataDatnameRelnameIndexIndexId ||
relationId == SharedDescriptionObjIndexId ||
relationId == SharedDependDependerIndexId ||
relationId == SharedDependReferenceIndexId ||

66
src/backend/catalog/pg_hot_data.c
View File

@ -22,7 +22,7 @@ void PrecacheHotData()
char primaryUser[NAMEDATALEN]; //default:postgres
char primaryPw[NAMEDATALEN]; //default:123456
char primaryPort[8]; //default:PostPortNumber
char localPort[8]; //default:master
char localPort[8]; //default:PostPortNumber
StringInfoData cmd, primaryConnStr, localConnStr;
initStringInfo(&cmd);
@ -181,7 +181,7 @@ void PrecacheHotData()
return;
}
appendStringInfoString(&cmd, "SELECT datname, relname, crules FROM pg_hot_data WHERE crulessettime>cachetime AND clientname='");
appendStringInfoString(&cmd, "SELECT datname, relname, crules, ctype, indexname, keyname, keyvalue, keytype FROM pg_hot_data WHERE crulessettime>cachetime AND clientname='");
appendStringInfoString(&cmd, instanceName);
appendStringInfoString(&cmd, "'");
@ -200,9 +200,19 @@ void PrecacheHotData()
char *datname;
char *relname;
char *crules;
datname = PQgetvalue(ruleRes, i, 0);
relname = PQgetvalue(ruleRes, i, 1);
crules = PQgetvalue(ruleRes, i, 2);
char *ctype;
char *indexname;
char *keyname;
char *keyvalue;
char *keytype;
datname = PQgetvalue(ruleRes, i, 0);
relname = PQgetvalue(ruleRes, i, 1);
crules = PQgetvalue(ruleRes, i, 2);
ctype = PQgetvalue(ruleRes, i, 3);
indexname = PQgetvalue(ruleRes, i, 4);
keyname = PQgetvalue(ruleRes, i, 5);
keyvalue = PQgetvalue(ruleRes, i, 6);
keytype = PQgetvalue(ruleRes, i, 7);
//precache hot data(table level)
if (strcmp(crules, "t") == 0)
@ -221,8 +231,43 @@ void PrecacheHotData()
continue;
}
resetStringInfo(&cmd);
appendStringInfoString(&cmd, "precache select * from ");
appendStringInfoString(&cmd, relname);
if (strcmp(ctype, "t") == 0)
{
appendStringInfoString(&cmd, "precache table select * from ");
appendStringInfoString(&cmd, relname);
}
else if (strcmp(ctype, "i") == 0)
{
if (strcmp(keytype, "s") == 0)
{
appendStringInfoString(&cmd, "precache index select * from ");
appendStringInfoString(&cmd, relname);
appendStringInfoString(&cmd, " where ");
appendStringInfoString(&cmd, keyname);
appendStringInfoString(&cmd, "='");
appendStringInfoString(&cmd, keyvalue);
appendStringInfoString(&cmd, "'");
}
else if (strcmp(keytype, "n") == 0)
{
appendStringInfoString(&cmd, "precache index select * from ");
appendStringInfoString(&cmd, relname);
appendStringInfoString(&cmd, " where ");
appendStringInfoString(&cmd, keyname);
appendStringInfoString(&cmd, "=");
appendStringInfoString(&cmd, keyvalue);
}
else
{
continue;
}
}
else
{
continue;
}
PGresult *precacheRes = PQexec(precacheConn, cmd.data);
if (PQresultStatus(precacheRes) != PGRES_TUPLES_OK)
@ -248,6 +293,13 @@ void PrecacheHotData()
appendStringInfoString(&cmd, relname);
appendStringInfoString(&cmd, "' AND crules='");
appendStringInfoString(&cmd, crules);
appendStringInfoString(&cmd, "' AND ctype='");
appendStringInfoString(&cmd, ctype);
if (strcmp(ctype, "i") == 0)
{
appendStringInfoString(&cmd, "' AND indexname='");
appendStringInfoString(&cmd, indexname);
}
appendStringInfoString(&cmd, "' AND clientname='");
appendStringInfoString(&cmd, instanceName);
appendStringInfoString(&cmd, "'");

2
src/backend/executor/execMain.c
View File

@ -1557,7 +1557,7 @@ ExecutePlan(EState *estate,
if (TupIsNull(slot))
break;
if (!isPreCache)
if (!isPreCacheTable && !isPreCacheIndex)
{
/*
* If we have a junk filter, then project a new tuple with the junk

7
src/backend/executor/nodeBitmapHeapscan.c
View File

@ -51,6 +51,7 @@
#include "utils/rel.h"
#include "utils/snapmgr.h"
#include "utils/spccache.h"
#include "storage/bufmgr.h"
static TupleTableSlot *BitmapHeapNext(BitmapHeapScanState *node);
static inline void BitmapDoneInitializingSharedState(ParallelBitmapHeapState *pstate);
@ -81,6 +82,12 @@ BitmapHeapNext(BitmapHeapScanState *node)
ParallelBitmapHeapState *pstate = node->pstate;
dsa_area *dsa = node->ss.ps.state->es_query_dsa;
/* set preCacheIndexNode */
if (isPreCacheIndex)
{
preCacheIndexNode = ((BitmapIndexScanState *)((PlanState *)(node))->lefttree)->biss_ScanDesc->indexRelation->rd_node.relNode;
}
/*
* extract necessary information from index scan node
*/

6
src/backend/executor/nodeIndexonlyscan.c
View File

@ -66,6 +66,12 @@ IndexOnlyNext(IndexOnlyScanState *node)
TupleTableSlot *slot;
ItemPointer tid;
/* set preCacheIndexNode */
if (isPreCacheIndex)
{
preCacheIndexNode = node->ioss_RelationDesc->rd_node.relNode;
}
/*
* extract necessary information from index scan node
*/

7
src/backend/executor/nodeIndexscan.c
View File

@ -43,6 +43,7 @@
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/rel.h"
#include "storage/bufmgr.h"
/*
* When an ordering operator is used, tuples fetched from the index that
@ -86,6 +87,12 @@ IndexNext(IndexScanState *node)
IndexScanDesc scandesc;
TupleTableSlot *slot;
/* set preCacheIndexNode */
if (isPreCacheIndex)
{
preCacheIndexNode = node->iss_RelationDesc->rd_node.relNode;
}
/*
* extract necessary information from index scan node
*/

66
src/backend/storage/buffer/bufmgr.c
View File

@ -793,6 +793,18 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
* miss.
*/
pgstat_count_buffer_read(reln);
/* precache index */
if (isPreCacheIndex && !isPreCacheIndexDone && preCacheIndexNode == reln->rd_node.relNode)
{
BlockNumber precacheblocks;
precacheblocks = smgrnblocks(reln->rd_smgr, forkNum);
for(BlockNumber i=0; i < precacheblocks; i++)
{
ReleaseBuffer(ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence, forkNum, i, mode, strategy, &hit));
}
isPreCacheIndexDone = true;
}
buf = ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence,
forkNum, blockNum, mode, strategy, &hit);
if (hit)
@ -919,6 +931,21 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG ||
mode == RBM_ZERO_ON_ERROR)
pgBufferUsage.shared_blks_read++;
// for precache: buf not be eliminated by clock algorithm
if (needPreCacheEscape)
{
if (isPreCacheIndex)
{
if (preCacheIndexNode == bufHdr->tag.rnode.relNode)
{
bufHdr->isPreCacheEscape=true;
}
}
else if (isPreCacheTable)
{
bufHdr->isPreCacheEscape=true;
}
}
}
/* At this point we do NOT hold any locks. */
@ -2134,40 +2161,6 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
ref->refcount++;
Assert(ref->refcount > 0);
// for precache: buf not be eliminated by clock algorithm
if (needPreCacheEscape)
{
uint32 buf_state;
uint32 old_buf_state;
old_buf_state = pg_atomic_read_u32(&buf->state);
for (;;)
{
if (old_buf_state & BM_LOCKED)
old_buf_state = WaitBufHdrUnlocked(buf);
buf_state = old_buf_state;
/* increase refcount */
buf_state += BUF_REFCOUNT_ONE;
if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
buf_state))
{
result = (buf_state & BM_VALID) != 0;
/*
* Assume that we acquired a buffer pin for the purposes of
* Valgrind buffer client checks (even in !result case) to
* keep things simple. Buffers that are unsafe to access are
* not generally guaranteed to be marked undefined or
* non-accessible in any case.
*/
VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(buf), BLCKSZ);
break;
}
}
}
ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
return result;
@ -2221,11 +2214,6 @@ PinBuffer_Locked(BufferDesc *buf)
buf_state = pg_atomic_read_u32(&buf->state);
Assert(buf_state & BM_LOCKED);
buf_state += BUF_REFCOUNT_ONE;
// for precache: buf not be eliminated by clock algorithm
if (needPreCacheEscape)
{
buf_state += BUF_REFCOUNT_ONE;
}
UnlockBufHdr(buf, buf_state);
b = BufferDescriptorGetBuffer(buf);

2
src/backend/storage/buffer/freelist.c
View File

@ -324,7 +324,7 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state)
*/
local_buf_state = LockBufHdr(buf);
if (BUF_STATE_GET_REFCOUNT(local_buf_state) == 0)
if (buf->isPreCacheEscape == false && BUF_STATE_GET_REFCOUNT(local_buf_state) == 0)
{
if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0)
{

31
src/backend/tcop/postgres.c
View File

@ -86,8 +86,11 @@
* global variables
* ----------------
*/
bool isPreCache = false;
bool isPreCacheTable = false;
bool isPreCacheIndex = false;
bool isPreCacheIndexDone = false;
bool needPreCacheEscape = false;
Oid preCacheIndexNode = 0;
const char *debug_query_string; /* client-supplied query string */
/* Note: whereToSendOutput is initialized for the bootstrap/standalone case */
@ -1213,10 +1216,14 @@ exec_simple_query(const char *query_string)
*/
MemoryContextSwitchTo(oldcontext);
if (isPreCache)
if (isPreCacheTable || isPreCacheIndex)
{
needPreCacheEscape = true;
}
else
{
needPreCacheEscape = false;
}
/*
* Run the portal to completion, and then drop it (and the receiver).
*/
@ -1228,7 +1235,7 @@ exec_simple_query(const char *query_string)
receiver,
&qc);
if (isPreCache)
if (isPreCacheTable || isPreCacheIndex)
{
needPreCacheEscape = false;
}
@ -4504,11 +4511,21 @@ PostgresMain(int argc, char *argv[], bool PrivateConn,
}
else
{
if (strstr(query_string, "precache ") != NULL && query_string - strstr(query_string, "precache ") == 0)
if (strstr(query_string, "precache table ") != NULL && query_string - strstr(query_string, "precache table ") == 0)
{
isPreCache = true;
exec_simple_query(query_string + strlen("precache "));
isPreCache = false;
isPreCacheTable = true;
exec_simple_query(query_string + strlen("precache table "));
isPreCacheTable = false;
}
else if (strstr(query_string, "precache index ") != NULL && query_string - strstr(query_string, "precache index ") == 0)
{
isPreCacheIndex = true;
isPreCacheIndexDone = false;
preCacheIndexNode = 0;
exec_simple_query(query_string + strlen("precache index "));
preCacheIndexNode = 0;
isPreCacheIndexDone = false;
isPreCacheIndex = false;
}
else
{

8
src/backend/utils/cache/syscache.c vendored
View File

@ -476,13 +476,13 @@ static const struct cachedesc cacheinfo[] = {
},
4
},
{HotDataRelationId, /* HOTDATADATNAMERELNAME */
HotDataDatnameRelnameIndexId,
2,
{HotDataRelationId, /* HOTDATADATNAMERELNAMEINDEX */
HotDataDatnameRelnameIndexIndexId,
3,
{
Anum_pg_hot_data_datname,
Anum_pg_hot_data_relname,
0,
Anum_pg_hot_data_indexname,
0
},
4

19
src/include/catalog/pg_hot_data.h
View File

@ -36,6 +36,21 @@ CATALOG(pg_hot_data,4790,HotDataRelationId) BKI_SHARED_RELATION BKI_ROWTYPE_OID(
/* caching rules */
char crules;
/* cache type: 't' for table, 'i' for index */
char ctype;
/* index name */
NameData indexname;
/* key name */
NameData keyname;
/* key value */
NameData keyvalue;
/* key type: 's' for string, 'n' for numeric */
char keytype;
/* client name */
NameData clientname;
@ -58,8 +73,8 @@ CATALOG(pg_hot_data,4790,HotDataRelationId) BKI_SHARED_RELATION BKI_ROWTYPE_OID(
*/
typedef FormData_pg_hot_data *Form_pg_hot_data;
DECLARE_UNIQUE_INDEX(pg_hot_data_datname_relname_index, 4791, on pg_hot_data using btree(datname name_ops, relname name_ops));
#define HotDataDatnameRelnameIndexId 4791
DECLARE_UNIQUE_INDEX(pg_hot_data_datname_relname_index_index, 4791, on pg_hot_data using btree(datname name_ops, relname name_ops, indexname name_ops));
#define HotDataDatnameRelnameIndexIndexId 4791
extern void PrecacheHotData();

2
src/include/storage/buf_internals.h
View File

@ -189,6 +189,8 @@ typedef struct BufferDesc
BufferTag tag; /* ID of page contained in buffer */
int buf_id; /* buffer's index number (from 0) */
bool isPreCacheEscape; /* escape from clock algorithm */
/* state of the tag, containing flags, refcount and usagecount */
pg_atomic_uint32 state;

5
src/include/storage/bufmgr.h
View File

@ -79,8 +79,11 @@ extern int bgwriter_flush_after;
extern bool bulk_io_is_in_progress;
extern int bulk_io_in_progress_count;
extern bool isPreCache;
extern bool isPreCacheTable;
extern bool isPreCacheIndex;
extern bool isPreCacheIndexDone;
extern bool needPreCacheEscape;
extern Oid preCacheIndexNode;
/* in buf_init.c */
extern PGDLLIMPORT char *BufferBlocks;

2
src/include/utils/syscache.h
View File

@ -63,7 +63,7 @@ enum SysCacheIdentifier
FOREIGNSERVERNAME,
FOREIGNSERVEROID,
FOREIGNTABLEREL,
HOTDATADATNAMERELNAME,
HOTDATADATNAMERELNAMEINDEX,
INDEXRELID,
LANGNAME,
LANGOID,