diff --git a/src/common/backend/catalog/pg_partition.cpp b/src/common/backend/catalog/pg_partition.cpp index 0069bb1c3ba33d8d580031f1c8e71f0ac76afa50..bbe22c01cd6b1bd2e49878410a300d6478d10a6d 100644 --- a/src/common/backend/catalog/pg_partition.cpp +++ b/src/common/backend/catalog/pg_partition.cpp @@ -2088,3 +2088,35 @@ bool PartExprKeyIsNull(Relation rel, char** partExprKeyStr) ReleaseSysCache(partTuple); return isnull; } + +Oid partid_get_rootid(Oid partoid, Oid *subparentid) +{ + Oid parentid; + Oid rootid; + + *subparentid = InvalidOid; + /* Search the partition's relcache entry */ + HeapTuple partitionTup = SearchSysCache1(PARTRELID, ObjectIdGetDatum(partoid)); + if (!HeapTupleIsValid(partitionTup)) { + return InvalidOid; + } + Form_pg_partition partitionForm = (Form_pg_partition)GETSTRUCT(partitionTup); + parentid = partitionForm->parentid; + if (partitionForm->parttype != PART_OBJ_TYPE_TABLE_SUB_PARTITION) { + rootid = parentid; + ReleaseSysCache(partitionTup); + return rootid; + } + ReleaseSysCache(partitionTup); + + /* subpartition branch */ + HeapTuple parentTup = SearchSysCache1(PARTRELID, ObjectIdGetDatum(parentid)); + if (!HeapTupleIsValid(parentTup)) { + return InvalidOid; + } + Form_pg_partition parentForm = (Form_pg_partition)GETSTRUCT(parentTup); + rootid = parentForm->parentid; + *subparentid = parentid; + ReleaseSysCache(parentTup); + return rootid; +} \ No newline at end of file diff --git a/src/common/backend/libpq/pqcomm.cpp b/src/common/backend/libpq/pqcomm.cpp index f185e80265a7690c074a0efde76bc18bf7a35082..eca569a5fbb32d6307cc287bdc472f8e65dd17e1 100644 --- a/src/common/backend/libpq/pqcomm.cpp +++ b/src/common/backend/libpq/pqcomm.cpp @@ -1237,9 +1237,13 @@ static int pq_recvbuf(void) int r; WaitState oldStatus = pgstat_report_waitstatus(STATE_WAIT_COMM); + + t_thrd.libpq_cxt.WorkerRecvStartupPacket = true; r = secure_read(u_sess->proc_cxt.MyProcPort, t_thrd.libpq_cxt.PqRecvBuffer + t_thrd.libpq_cxt.PqRecvLength, PQ_RECV_BUFFER_SIZE - t_thrd.libpq_cxt.PqRecvLength); + t_thrd.libpq_cxt.WorkerRecvStartupPacket = false; + (void)pgstat_report_waitstatus(oldStatus); if (r < 0) { diff --git a/src/gausskernel/optimizer/commands/vacuum.cpp b/src/gausskernel/optimizer/commands/vacuum.cpp index c90340a3784321d37ee931e1d327cca5f6fd4047..1414cca908f634ea8692c85e13077531f30cd4f2 100644 --- a/src/gausskernel/optimizer/commands/vacuum.cpp +++ b/src/gausskernel/optimizer/commands/vacuum.cpp @@ -44,6 +44,7 @@ #include "catalog/pgxc_class.h" #include "catalog/storage.h" #include "catalog/storage_gtt.h" +#include "catalog/pg_partition_fn.h" #include "commands/cluster.h" #include "commands/dbcommands.h" #include "commands/matview.h" @@ -2018,16 +2019,11 @@ static bool vacuum_rel(Oid relid, VacuumStmt* vacstmt, bool do_toast) /* Get the partitioned table's oid */ if (vacuumPartition(vacstmt->flags)) { - relationid = partid_get_parentid(relid); + relationid = partid_get_rootid(relid, &subparentid); if (!OidIsValid(relationid)) { proc_snapshot_and_transaction(); return false; } - Oid grandparentid = partid_get_parentid(relationid); - if (OidIsValid(grandparentid)) { - subparentid = relationid; - relationid = grandparentid; - } } /* diff --git a/src/gausskernel/process/postmaster/autovacuum.cpp b/src/gausskernel/process/postmaster/autovacuum.cpp index b7fba05c9bf7292b99c9281c4bb327b8ff1a36fc..10228db977cd4774bc3ccf0ee190a9d1a5264e59 100755 --- a/src/gausskernel/process/postmaster/autovacuum.cpp +++ b/src/gausskernel/process/postmaster/autovacuum.cpp @@ -2740,18 +2740,18 @@ static void do_autovacuum(void) * vacuum and analyze in different transactions. */ if (vacuumPartition((uint32)(vacObj->flags))) { - Oid at_parentid = partid_get_parentid(tab->at_relid); - Oid at_grandparentid = partid_get_parentid(at_parentid); - if (OidIsValid(at_grandparentid)) { + Oid subparentid = InvalidOid; + Oid tableoid = partid_get_rootid(tab->at_relid, &subparentid); + if (OidIsValid(subparentid)) { tab->at_subpartname = getPartitionName(tab->at_relid, false); tab->at_partname = NULL; - tab->at_relname = get_rel_name(at_grandparentid); - tab->at_nspname = get_namespace_name(get_rel_namespace(at_grandparentid)); + tab->at_relname = get_rel_name(tableoid); + tab->at_nspname = get_namespace_name(get_rel_namespace(tableoid)); } else { tab->at_subpartname = NULL; tab->at_partname = getPartitionName(tab->at_relid, false); - tab->at_relname = get_rel_name(at_parentid); - tab->at_nspname = get_namespace_name(get_rel_namespace(at_parentid)); + tab->at_relname = get_rel_name(tableoid); + tab->at_nspname = get_namespace_name(get_rel_namespace(tableoid)); } } else { tab->at_subpartname = NULL; @@ -2805,11 +2805,7 @@ static void do_autovacuum(void) if (vacObj->flags & VACFLG_SUB_PARTITION) { // Get partitioned/subpartitioned table's oid - Oid table_oid = parentid; - Oid grandparentid = partid_get_parentid(parentid); - if (OidIsValid(grandparentid)) { - table_oid = grandparentid; - } + Oid table_oid = partid_get_rootid(relid, &parentid); // Update ap_entry->at_gpivacuumed ap_entry = (at_partitioned_table*)hash_search(partitioned_tables_map, &table_oid, HASH_FIND, &found); if (found && !ap_entry->at_gpivacuumed && tab->at_gpivacuumed) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index fc7f427f7a3f49d335e0f41df0479fbe69f0f4e9..6829aa9b586e6a30a3a9d366f41bf903676c1e20 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -11268,6 +11268,11 @@ void startup_die(SIGNAL_ARGS) } else { proc_exit(1); } + if (t_thrd.role == WORKER && t_thrd.libpq_cxt.WorkerRecvStartupPacket) { + int sock = u_sess->proc_cxt.MyProcPort->sock; + u_sess->proc_cxt.MyProcPort->sock = -1; + closesocket(sock); + } } /* copy from startup_die, and set cancel_from_timeout flag */ diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 7de04d2dedc80b61e51e9badbd0997fa8a9865eb..64eed703987cffdd1bf35e3c227cf2fb19be89c5 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1293,6 +1293,7 @@ static void knl_t_libpq_init(knl_t_libpq_context* libpq_cxt) libpq_cxt->parsed_hba_lines = NIL; libpq_cxt->parsed_hba_context = NULL; + libpq_cxt->WorkerRecvStartupPacket = false; } diff --git a/src/gausskernel/runtime/executor/execMain.cpp b/src/gausskernel/runtime/executor/execMain.cpp index bd73ef320566d43a65e5e2d8f6866af7328210bc..6022b6da02d5589d550d3c8f0906d16e2980b75d 100755 --- a/src/gausskernel/runtime/executor/execMain.cpp +++ b/src/gausskernel/runtime/executor/execMain.cpp @@ -3774,8 +3774,10 @@ void EvalPlanQualFetchRowMarks(EPQState *epqstate) Partition p = partitionOpen(erm->relation, tableoid, NoLock, bucketid); Relation fakeRelation = partitionGetRelation(erm->relation, p); + Snapshot snapshot = tableam_eval_planqual_fetch_snapshot(fakeRelation, epqstate); + /* okay, fetch the tuple */ - if (!tableam_tuple_fetch(fakeRelation, SnapshotAny, &tuple, &buffer, false, NULL)) { + if (!tableam_tuple_fetch(fakeRelation, snapshot, &tuple, &buffer, false, NULL)) { ereport(ERROR, (errcode(ERRCODE_FETCH_DATA_FAILED), errmsg("failed to fetch tuple for EvalPlanQual recheck from partition relation."))); } @@ -3788,7 +3790,10 @@ void EvalPlanQualFetchRowMarks(EPQState *epqstate) Assert(bucketid != InvalidBktId); fakeRelation = bucketGetRelation(erm->relation, NULL, bucketid); } - if (!tableam_tuple_fetch(fakeRelation, SnapshotAny, &tuple, &buffer, true, NULL)) { + + Snapshot snapshot = tableam_eval_planqual_fetch_snapshot(fakeRelation, epqstate); + + if (!tableam_tuple_fetch(fakeRelation, snapshot, &tuple, &buffer, true, NULL)) { Page page = BufferGetPage(buffer); ItemPointer tid = &tuple.t_self; OffsetNumber offnum = ItemPointerGetOffsetNumber(tid); diff --git a/src/gausskernel/storage/access/heap/tuptoaster.cpp b/src/gausskernel/storage/access/heap/tuptoaster.cpp index b405a8954b479d23e4724ff098ab657a87b04074..181aa149d825675578bf3680dabd90d63dcdebd2 100644 --- a/src/gausskernel/storage/access/heap/tuptoaster.cpp +++ b/src/gausskernel/storage/access/heap/tuptoaster.cpp @@ -612,6 +612,7 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu * ' ' default handling * 'p' already processed --- don't touch it * 'x' incompressible, but OK to move off + * 'l' logical wal, need save toast no-modified columns * * NOTE: toast_sizes[i] is only made valid for varlena attributes with * toast_action[i] different from 'p'. @@ -644,7 +645,7 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu */ if (att[i].attlen == -1 && !toast_oldisnull[i] && (VARATT_IS_EXTERNAL_ONDISK_B(old_value) || VARATT_IS_HUGE_TOAST_POINTER(old_value))) { - if (toast_isnull[i] || (RelationIsLogicallyLogged(rel) && !VARATT_IS_HUGE_TOAST_POINTER(new_value)) || + if (toast_isnull[i] || !(VARATT_IS_EXTERNAL_ONDISK_B(new_value) || VARATT_IS_HUGE_TOAST_POINTER(new_value)) || VARTAG_EXTERNAL(new_value) != VARTAG_EXTERNAL(old_value) || memcmp((char *)old_value, (char *)new_value, VARSIZE_EXTERNAL(old_value)) != 0) { @@ -654,6 +655,16 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu */ toast_delold[i] = true; need_delold = true; + } else if (RelationIsLogicallyLogged(rel) && !VARATT_IS_HUGE_TOAST_POINTER(new_value)) { + /* + * Logical decode needs to track 'delete' and 'insert' actions, even if the column is not modified. + * We just do the same toast action as how it stores before and don't care about current attstorage. + * For example: attstorage is modified form 'x' to 'p', if we store as 'p' now, it may exceed + * the block size because toast doesn't process attstorage 'p'. + */ + toast_delold[i] = true; + need_delold = true; + toast_action[i] = 'l'; } else { /* * This attribute isn't changed by this update so we reuse @@ -687,7 +698,7 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu /* * If the table's attribute says PLAIN always, force it so. */ - if (att[i].attstorage == 'p') { + if (att[i].attstorage == 'p' && toast_action[i] != 'l') { toast_action[i] = 'p'; } @@ -700,7 +711,7 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu */ if (VARATT_IS_EXTERNAL(new_value) && !VARATT_IS_HUGE_TOAST_POINTER(new_value)) { toast_oldexternal[i] = new_value; - if (att[i].attstorage == 'p') { + if (att[i].attstorage == 'p' && toast_action[i] != 'l') { new_value = heap_tuple_untoast_attr(new_value); } else { new_value = heap_tuple_fetch_attr(new_value); @@ -731,6 +742,24 @@ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtu } } + /* For unmodified toast columns, save toast if wal level is logical */ + for (i = 0; i < num_attrs; i++) { + if (toast_action[i] != 'l') { + continue; + } + + /* toast external store */ + Datum old_value = toast_values[i]; + toast_values[i] = toast_save_datum(rel, toast_values[i], toast_oldexternal[i], options); + if (toast_free[i]) { + pfree(DatumGetPointer(old_value)); + } + toast_free[i] = true; + need_change = true; + need_free = true; + toast_action[i] = 'p'; + } + /* * Compress and/or save external until data fits into target length * diff --git a/src/gausskernel/storage/access/table/tableam.cpp b/src/gausskernel/storage/access/table/tableam.cpp index 755725bdc30de4f0f46fc75e0a85090c65158b0c..ab1753a390937948d694c8c4d2cf3c4ba0bb8f54 100644 --- a/src/gausskernel/storage/access/table/tableam.cpp +++ b/src/gausskernel/storage/access/table/tableam.cpp @@ -424,10 +424,15 @@ bool HeapamTupleCheckCompress(Tuple tuple) return HEAP_TUPLE_IS_COMPRESSED(((HeapTuple)tuple)->t_data); } +Snapshot HeapamTupleFetchEpqSnapshot(EPQState * /* epqstate */) +{ + return SnapshotAny; +} + void HeapamTcapPromoteLock(Relation relation, LOCKMODE *lockmode) { - /* Protect old versions from recycling during timecapsule. */ - *lockmode = AccessExclusiveLock; + /* astore engine not support tcap now and do nothing. Need block autovacuum if support in future version. */ + return; } bool HeapamTcapValidateSnap(Relation relation, Snapshot snap) @@ -548,6 +553,7 @@ static const TableAmRoutine g_heapam_methods = { tuple_lock_updated : HeapamTupleLockUpdated, tuple_check_visible: HeapamTupleCheckVisible, tuple_abort_speculative: HeapamAbortSpeculative, + tuple_fetch_epq_snapshot : HeapamTupleFetchEpqSnapshot, tuple_check_compress: HeapamTupleCheckCompress, /* ------------------------------------------------------------------------ @@ -568,6 +574,11 @@ static const TableAmRoutine g_heapam_methods = { tcap_insert_lost : HeapamTcapInsertLost }; +Snapshot tableam_eval_planqual_fetch_snapshot(Relation relation, EPQState *epqstate) +{ + return relation->rd_tam_ops->tuple_fetch_epq_snapshot(epqstate); +} + /* * Implementation of uheap accessor methods. */ @@ -1119,6 +1130,15 @@ bool UHeapamTupleCheckCompress(Tuple tuple) return false; } +Snapshot UHeapamFetchEpqSnapshot(EPQState *epqstate) +{ + /* + * Try to fetch the row with mvcc snapshot in ustore epq to prevent it from being cleared + * by another session. + */ + return epqstate->parentestate->es_snapshot; +} + void UHeapamTupleCheckVisible(Snapshot snapshot, Tuple tuple, Buffer buffer) { UHeapTupleCheckVisible(snapshot, (UHeapTuple)tuple, buffer); @@ -1252,6 +1272,7 @@ static const TableAmRoutine g_ustoream_methods = { tuple_lock_updated : UHeapamTupleLockUpdated, tuple_check_visible : UHeapamTupleCheckVisible, tuple_abort_speculative : UHeapamAbortSpeculative, + tuple_fetch_epq_snapshot : UHeapamFetchEpqSnapshot, tuple_check_compress: UHeapamTupleCheckCompress, /* ------------------------------------------------------------------------ diff --git a/src/gausskernel/storage/access/transam/double_write.cpp b/src/gausskernel/storage/access/transam/double_write.cpp index bbfcb2788407817e9fe4728afe260be863a0dc94..dfed766b7f98fa5cd4eb939ca590417db9a13798 100644 --- a/src/gausskernel/storage/access/transam/double_write.cpp +++ b/src/gausskernel/storage/access/transam/double_write.cpp @@ -2241,7 +2241,6 @@ static void dw_batch_flush(dw_batch_file_context *dw_cxt, XLogRecPtr latest_lsn, /* used to block the io for snapshot feature */ (void)LWLockAcquire(g_instance.ckpt_cxt_ctl->snapshotBlockLock, LW_SHARED); - LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); if (!XLogRecPtrIsInvalid(latest_lsn)) { XLogWaitFlush(latest_lsn); @@ -2274,6 +2273,7 @@ static void dw_batch_flush(dw_batch_file_context *dw_cxt, XLogRecPtr latest_lsn, dw_cxt->write_pos = 0; thrd_dw_cxt->dw_page_idx = offset_page; LWLockRelease(dw_cxt->flush_lock); + LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); ereport(DW_LOG_LEVEL, (errmodule(MOD_DW), diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 553c800293c969dbf559b2edcf229741db89cad3..452ec1f0e60b88582f3ca9eee0afd883faf71307 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -600,6 +600,10 @@ static void StartPageRedoWorkers(uint32 totalThrdNum) RedoRoleInit(&(g_dispatcher->readLine.readPageThd), tmpWorkers[workerCnt++], REDO_READ_PAGE_WORKER, 0, false); RedoRoleInit(&(g_dispatcher->readLine.readThd), tmpWorkers[workerCnt++], REDO_READ_WORKER, 0, false); + g_GlobalLsnForwarder.record.ReadRecPtr = InvalidXLogRecPtr; + g_GlobalLsnForwarder.record.EndRecPtr = InvalidXLogRecPtr; + g_GlobalLsnForwarder.record.refcount = 0; + for (started = 0; started < totalThrdNum; started++) { if (StartPageRedoWorker(g_dispatcher->allWorkers[started]) == NULL) { ereport(PANIC, diff --git a/src/gausskernel/storage/access/transam/single_double_write.cpp b/src/gausskernel/storage/access/transam/single_double_write.cpp index b11c277497c276bd2eb97bf2034ca0e7aab64934..24c2aa780d45272c3d75dabdc701953b322d8390 100644 --- a/src/gausskernel/storage/access/transam/single_double_write.cpp +++ b/src/gausskernel/storage/access/transam/single_double_write.cpp @@ -671,7 +671,6 @@ uint16 first_version_dw_single_flush(BufferDesc *buf_desc) /* used to block the io for snapshot feature */ (void)LWLockAcquire(g_instance.ckpt_cxt_ctl->snapshotBlockLock, LW_SHARED); - LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); uint64 buf_state = LockBufHdr(buf_desc); Block block = BufHdrGetBlock(buf_desc); @@ -713,6 +712,7 @@ uint16 first_version_dw_single_flush(BufferDesc *buf_desc) dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, page_write_offset, SINGLE_DW_FILE_NAME); (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.total_writes, 1); + LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); return actual_pos; } @@ -732,7 +732,6 @@ uint16 second_version_dw_single_flush(BufferTag tag, Block block, XLogRecPtr pag /* used to block the io for snapshot feature */ (void)LWLockAcquire(g_instance.ckpt_cxt_ctl->snapshotBlockLock, LW_SHARED); - LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); /* first step, copy buffer to dw buf, than flush page lsn, the buffer content lock is already held */ rc = memcpy_s(buf, BLCKSZ, block, BLCKSZ); @@ -775,6 +774,7 @@ uint16 second_version_dw_single_flush(BufferTag tag, Block block, XLogRecPtr pag LWLockRelease(dw_single_cxt->second_buftag_lock); (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.second_total_writes, 1); + LWLockRelease(g_instance.ckpt_cxt_ctl->snapshotBlockLock); return (actual_pos + DW_FIRST_DATA_PAGE_NUM); } diff --git a/src/gausskernel/storage/access/ubtree/ubtrecycle.cpp b/src/gausskernel/storage/access/ubtree/ubtrecycle.cpp index 630eebb455967f981381e7be3217ce8ffa1a2a04..f575861fcc5a26af40e737af6ad7995f02487c86 100644 --- a/src/gausskernel/storage/access/ubtree/ubtrecycle.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtrecycle.cpp @@ -31,6 +31,8 @@ #include "utils/builtins.h" #include "datatype/timestamp.h" +#define UBTREE_NBLOCKS_DIFF 24 + uint32 BlockGetMaxItems(BlockNumber blkno) { uint32 freeSpace = BLCKSZ - sizeof(PageHeaderData) - offsetof(UBTRecycleQueueHeaderData, items); @@ -445,6 +447,14 @@ Buffer UBTreeGetAvailablePage(Relation rel, UBTRecycleForkNumber forkNumber, UBT Buffer metaBuf = ReadRecycleQueueBuffer(rel, metaBlockNumber); LockBuffer(metaBuf, BT_READ); UBTRecycleMeta metaData = (UBTRecycleMeta)PageGetContents(BufferGetPage(metaBuf)); + BlockNumber meta_block_upper = 0; + if (nblocks > UBTREE_NBLOCKS_DIFF) { + meta_block_upper = (nblocks - UBTREE_NBLOCKS_DIFF); + } + if (metaData->nblocksUpper < meta_block_upper) { + metaData->nblocksUpper = meta_block_upper; + metaChanged = true; + } for (BlockNumber curBlkno = metaData->nblocksUpper; curBlkno < nblocks; curBlkno++) { if (t_thrd.int_cxt.QueryCancelPending || t_thrd.int_cxt.ProcDiePending) { ereport(ERROR, (errmsg("Received cancel interrupt while getting available page."))); diff --git a/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp b/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp index e3b58a83e6fc4b12367aa8b5fd38a9da1e0553b2..c4e57c8c12a49c6b6aee8f32db52a151d1473f46 100644 --- a/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp +++ b/src/gausskernel/storage/access/ustore/knl_utuptoaster.cpp @@ -196,7 +196,7 @@ UHeapTuple UHeapToastInsertOrUpdate(Relation relation, UHeapTuple newtup, UHeapT * we have to delete it later. */ if (att->attlen == -1 && !toastOldIsNull[i] && VARATT_IS_EXTERNAL_ONDISK(oldValue)) { - if (toastIsNull[i] || !VARATT_IS_EXTERNAL_ONDISK(newValue) || RelationIsLogicallyLogged(relation) || + if (toastIsNull[i] || !VARATT_IS_EXTERNAL_ONDISK(newValue) || memcmp((char *)oldValue, (char *)newValue, VARSIZE_EXTERNAL(oldValue)) != 0) { /* * The old external stored value isn't needed any more @@ -204,6 +204,16 @@ UHeapTuple UHeapToastInsertOrUpdate(Relation relation, UHeapTuple newtup, UHeapT */ toastDelOld[i] = true; needDelOld = true; + } else if (RelationIsLogicallyLogged(relation)) { + /* + * Logical decode needs to track 'delete' and 'insert' actions, even if the column is not modified. + * We just do the same toast action as how it stores before and don't care about current attstorage. + * For example: attstorage is modified form 'x' to 'p', if we store as 'p' now, it may exceed + * the block size because toast doesn't process attstorage 'p'. + */ + needDelOld = true; + toastDelOld[i] = true; + toastAction[i] = 'l'; } else { /* * This attribute isn't changed by this update so we reuse @@ -238,7 +248,7 @@ UHeapTuple UHeapToastInsertOrUpdate(Relation relation, UHeapTuple newtup, UHeapT /* * If the table's attribute says PLAIN always, force it so. */ - if (att->attstorage == 'p') + if (att->attstorage == 'p' && toastAction[i] != 'l') toastAction[i] = 'p'; /* @@ -251,7 +261,7 @@ UHeapTuple UHeapToastInsertOrUpdate(Relation relation, UHeapTuple newtup, UHeapT */ if (VARATT_IS_EXTERNAL(newValue)) { toastOldExternal[i] = newValue; - if (att->attstorage == 'p') + if (att->attstorage == 'p' && toastAction[i] != 'l') newValue = heap_tuple_untoast_attr(newValue); else newValue = heap_tuple_fetch_attr(newValue); @@ -273,6 +283,24 @@ UHeapTuple UHeapToastInsertOrUpdate(Relation relation, UHeapTuple newtup, UHeapT } } + /* For unmodified toast columns, save toast if wal level is logical */ + for (i = 0; i < numAttrs; i++) { + if (toastAction[i] != 'l') { + continue; + } + + /* toast external store */ + Datum oldValue = toastValues[i]; + toastValues[i] = UHeapToastSaveDatum(relation, toastValues[i], toastOldExternal[i], options); + if (toastFree[i]) { + pfree(DatumGetPointer(oldValue)); + } + toastFree[i] = true; + needChange = true; + needFree = true; + toastAction[i] = 'p'; + } + /* ---------- * Compress and/or save external until data fits into target length * diff --git a/src/gausskernel/storage/ipc/standby.cpp b/src/gausskernel/storage/ipc/standby.cpp index c9680e6ed32dedeef1ba6e28bcc52be3042ee653..2c598cc49046dbf97e0f9e7e653eba09e93c82ed 100755 --- a/src/gausskernel/storage/ipc/standby.cpp +++ b/src/gausskernel/storage/ipc/standby.cpp @@ -24,6 +24,7 @@ #include "access/xloginsert.h" #include "access/csnlog.h" #include "access/multi_redo_api.h" +#include "access/double_write.h" #include "miscadmin.h" #include "storage/buf/bufmgr.h" #include "storage/lmgr.h" @@ -136,7 +137,8 @@ static TimestampTz GetStandbyLimitTime(TimestampTz startTime) { TimestampTz rtime = startTime; - if (u_sess->attr.attr_storage.max_standby_streaming_delay < 0) + if (u_sess->attr.attr_storage.max_standby_streaming_delay < 0 || + is_dw_snapshot_blocked()) return 0; /* wait forever */ return TimestampTzPlusMilliseconds(rtime, u_sess->attr.attr_storage.max_standby_streaming_delay); diff --git a/src/gausskernel/storage/replication/archive_walreceiver.cpp b/src/gausskernel/storage/replication/archive_walreceiver.cpp index 6af2c0b3f27be2bebe36220b9614b03df4131fcd..6a82950b1b42d58a848620ed7aa9a695623d9494 100644 --- a/src/gausskernel/storage/replication/archive_walreceiver.cpp +++ b/src/gausskernel/storage/replication/archive_walreceiver.cpp @@ -861,6 +861,11 @@ int archive_replication_cleanup(XLogRecPtr recptr, ArchiveConfig *archive_config size_t len = 0; XLogSegNo xlogSegno = 0; volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; + struct timeval start_time; + gettimeofday(&start_time, NULL); + struct timeval curr_time; + /* avoid taking too long to delete obs file, set timeout as 4h */ + long timeout = 4 * 60 * 60; XLByteToSeg(recptr, xlogSegno); rc = snprintf_s(xlogfname, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X_%02u", DEFAULT_TIMELINE_ID, @@ -909,6 +914,18 @@ int archive_replication_cleanup(XLogRecPtr recptr, ArchiveConfig *archive_config } foreach (cell, object_list) { + /* check interrupts */ + CHECK_FOR_INTERRUPTS(); + + gettimeofday(&curr_time, NULL); + if (curr_time.tv_sec - start_time.tv_sec > timeout) { + /* release result list */ + list_free_deep(object_list); + object_list = NIL; + pfree(fileNamePrefix); + ereport(ERROR, (errmsg("archive_replication_cleanup failed, timeout %ld s, current delete file is %s", + timeout, (char *)lfirst(cell)))); + } key = path_skip_prefix((char *)lfirst(cell)); if (key == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index bf30cea8e8b95b57898d30deb4dc54377a52acb2..b155e23851f2a8b8e7ee1a8eec6072a49497dead 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -503,6 +503,8 @@ typedef struct TableAmRoutine { void (*tuple_abort_speculative)(Relation relation, Tuple tuple); + Snapshot (*tuple_fetch_epq_snapshot)(EPQState *epqstate); + bool (*tuple_check_compress)(Tuple tuple); /* ------------------------------------------------------------------------ @@ -533,6 +535,7 @@ typedef struct TableAmRoutine { extern const TableAmRoutine * const g_tableam_routines[]; extern void HeapamScanIndexFetchEnd(IndexFetchTableData *scan); extern void heapam_index_fetch_reset(IndexFetchTableData *scan); +extern Snapshot tableam_eval_planqual_fetch_snapshot(Relation relation, EPQState *epqstate); extern IndexFetchTableData *HeapamScanIndexFetchBegin(Relation rel); static inline const TableAmRoutine* GetTableAmRoutine(TableAmType type) diff --git a/src/include/catalog/pg_partition_fn.h b/src/include/catalog/pg_partition_fn.h index 4f340f252332fce01e6010e29901144a01829d45..7cfc1a50866f27372133228657bbea08efc1d672 100644 --- a/src/include/catalog/pg_partition_fn.h +++ b/src/include/catalog/pg_partition_fn.h @@ -268,6 +268,7 @@ extern void releasePartitionList(Relation relation, List** partList, LOCKMODE l extern void releaseSubPartitionList(Relation relation, List** partList, LOCKMODE lockmode); extern void releasePartitionOidList(List** partList); extern void ReleaseSubPartitionOidList(List** partList); +extern Oid partid_get_rootid(Oid partoid, Oid *subparentid); extern bool PartExprKeyIsNull(Relation rel, char** partExprKeyStr = NULL); #endif diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 93c7cd17274c30ebb8571c9040c51abcd7533106..7359c8c9b58e6b04c33cfe24cf073656f7c98eef 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -2213,6 +2213,7 @@ typedef struct knl_t_libpq_context { /* variables for save query results to temp file*/ bool save_query_result_to_disk; + bool WorkerRecvStartupPacket; struct TempFileContextInfo* PqTempFileContextInfo; /*