Merge pull request #678 from arenadata/6.26.2-sync

Sync 6.26.2 changes

.abi-check/6.26.0_arenadata54/postgres.symbols.ignore

@@ -0,0 +1,2 @@
+ConfigureNamesBool_gp
+ConfigureNamesInt_gp

.abi-check/6.26.1/postgres.symbols.ignore

@@ -0,0 +1 @@
+ConfigureNamesBool_gp

GNUmakefile.in

@@ -24,6 +24,7 @@ all:
 	$(MAKE) -C contrib/pg_trgm all
 	$(MAKE) -C contrib/btree_gin all
 	$(MAKE) -C contrib/isn all
+	$(MAKE) -C contrib/pg_buffercache all
 ifeq ($(OS),Darwin)
 	@echo "dblink and postgres_fdw can't be built on Mac"
 else
@@ -78,6 +79,7 @@ install:
 	$(MAKE) -C contrib/pg_trgm $@
 	$(MAKE) -C contrib/btree_gin $@
 	$(MAKE) -C contrib/isn $@
+	$(MAKE) -C contrib/pg_buffercache $@
 ifneq ($(OS),Darwin)
 	$(MAKE) -C contrib/dblink $@
 	$(MAKE) -C contrib/postgres_fdw $@
@@ -180,6 +182,7 @@ ICW_TARGETS += contrib/pg_stat_statements
 ICW_TARGETS += contrib/formatter_fixedwidth
 ICW_TARGETS += contrib/extprotocol
 ICW_TARGETS += contrib/pg_trgm contrib/btree_gin contrib/isn
+ICW_TARGETS += contrib/pg_buffercache
 ifneq ($(OS),Darwin)
 	ICW_TARGETS += contrib/dblink contrib/postgres_fdw
 endif

concourse/scripts/compile_gpdb.bash

@@ -95,10 +95,13 @@ function include_dependencies() {
 	header_search_path=( /usr/local/include/ /usr/include/ )
 	vendored_headers=(zstd*.h uv.h uv )
 	pkgconfigs=(libzstd.pc libuv.pc quicklz.pc)
-	# rocky9/oel9/rhel9 won't vendor zstd because of rsync on these platofrm does not work libzstd 1.3.7
+	# rocky9/oel9/rhel9 won't vendor zstd because of rsync on these platform does not work libzstd 1.3.7
 	if [[ ${BLD_ARCH} == "rhel9"* ]]; then
 	  vendored_libs=(libquicklz.so{,.1,.1.5.0} libuv.so{,.1,.1.0.0} libxerces-c{,-3.1}.so)
-	else
+	# rocky8/oel8/rhel8 needs zstd 1.4.4 to be vendor because these platform support system libzstd 1.4.4
+	elif [[ ${BLD_ARCH} == "rhel8"* ]]; then
+	  vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.4.4} libuv.so{,.1,.1.0.0} libxerces-c{,-3.1}.so)
+  else
 	vendored_libs=(libquicklz.so{,.1,.1.5.0} libzstd.so{,.1,.1.3.7} libuv.so{,.1,.1.0.0} libxerces-c{,-3.1}.so)
 	fi
 

contrib/pg_buffercache/.gitignore

@@ -1 +1,4 @@
-/pg_buffercache.sql
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/

contrib/pg_buffercache/Makefile

@@ -4,7 +4,10 @@ MODULE_big = pg_buffercache
 OBJS = pg_buffercache_pages.o
 
 EXTENSION = pg_buffercache
-DATA = pg_buffercache--1.0.sql pg_buffercache--unpackaged--1.0.sql
+DATA = pg_buffercache--1.1.sql pg_buffercache--1.0--1.1.sql pg_buffercache--unpackaged--1.0.sql
+PGFILEDESC = "pg_buffercache - monitoring of shared buffer cache in real-time"
+
+REGRESS = pg_buffercache
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config

contrib/pg_buffercache/expected/pg_buffercache.out

@@ -0,0 +1,33 @@
+CREATE EXTENSION pg_buffercache;
+select count(*) = (select setting::bigint
+                   from pg_settings
+                   where name = 'shared_buffers')
+from pg_buffercache;
+ ?column? 
+----------
+ t
+(1 row)
+
+SELECT count(*) = (select setting::bigint
+                   from pg_settings
+                   where name = 'shared_buffers') *
+                   (select count(*) from gp_segment_configuration where role='p')
+                   as buffers
+FROM gp_buffercache;
+ buffers 
+---------
+ t
+(1 row)
+
+-- Check that the functions / views can't be accessed by default.
+CREATE ROLE buffercache_test;
+NOTICE:  resource queue required -- using default resource queue "pg_default"
+SET ROLE buffercache_test;
+SELECT * FROM pg_buffercache;
+ERROR:  permission denied for relation pg_buffercache
+SELECT * FROM pg_buffercache_pages() AS p (wrong int);
+ERROR:  permission denied for function pg_buffercache_pages
+SELECT * FROM gp_buffercache;
+ERROR:  permission denied for relation gp_buffercache
+RESET ROLE;
+DROP ROLE buffercache_test;

contrib/pg_buffercache/pg_buffercache--1.0--1.1.sql

@@ -0,0 +1,21 @@
+/* contrib/pg_buffercache/pg_buffercache--1.0--1.1.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION pg_buffercache UPDATE TO '1.1'" to load this file. \quit
+
+-- Upgrade view to 1.1. format
+CREATE OR REPLACE VIEW pg_buffercache AS
+	SELECT P.* FROM pg_buffercache_pages() AS P
+	(bufferid integer, relfilenode oid, reltablespace oid, reldatabase oid,
+	 relforknumber int2, relblocknumber int8, isdirty bool, usagecount int2,
+	 pinning_backends int4);
+
+CREATE VIEW gp_buffercache AS
+	SELECT gp_execution_segment() AS gp_segment_id, *
+	FROM gp_dist_random('pg_buffercache')
+	UNION ALL
+	SELECT -1 AS gp_segment_id, *
+  FROM pg_buffercache
+  ORDER BY 1,2;
+
+REVOKE ALL ON gp_buffercache FROM PUBLIC;

contrib/pg_buffercache/pg_buffercache--1.0.sql → contrib/pg_buffercache/pg_buffercache--1.1.sql

@@ -1,4 +1,4 @@
-/* contrib/pg_buffercache/pg_buffercache--1.0.sql */
+/* contrib/pg_buffercache/pg_buffercache--1.1.sql */
 
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION pg_buffercache" to load this file. \quit
@@ -13,8 +13,18 @@ LANGUAGE C;
 CREATE VIEW pg_buffercache AS
 	SELECT P.* FROM pg_buffercache_pages() AS P
 	(bufferid integer, relfilenode oid, reltablespace oid, reldatabase oid,
-	 relforknumber int2, relblocknumber int8, isdirty bool, usagecount int2);
+	 relforknumber int2, relblocknumber int8, isdirty bool, usagecount int2,
+	 pinning_backends int4);
+
+CREATE VIEW gp_buffercache AS
+	SELECT gp_execution_segment() AS gp_segment_id, *
+	FROM gp_dist_random('pg_buffercache')
+	UNION ALL
+	SELECT -1 AS gp_segment_id, *
+  FROM pg_buffercache
+  ORDER BY 1,2;
 
 -- Don't want these to be available to public.
 REVOKE ALL ON FUNCTION pg_buffercache_pages() FROM PUBLIC;
 REVOKE ALL ON pg_buffercache FROM PUBLIC;
+REVOKE ALL ON gp_buffercache FROM PUBLIC;

contrib/pg_buffercache/pg_buffercache.control

@@ -1,5 +1,5 @@
 # pg_buffercache extension
 comment = 'examine the shared buffer cache'
-default_version = '1.0'
+default_version = '1.1'
 module_pathname = '$libdir/pg_buffercache'
 relocatable = true

contrib/pg_buffercache/pg_buffercache_pages.c

@@ -15,7 +15,8 @@
 #include "storage/bufmgr.h"
 
 
-#define NUM_BUFFERCACHE_PAGES_ELEM	8
+#define NUM_BUFFERCACHE_PAGES_MIN_ELEM	8
+#define NUM_BUFFERCACHE_PAGES_ELEM	9
 
 PG_MODULE_MAGIC;
 
@@ -34,6 +35,12 @@ typedef struct
 	bool		isvalid;
 	bool		isdirty;
 	uint16		usagecount;
+	/*
+	 * An int32 is sufficiently large, as MAX_BACKENDS prevents a buffer from
+	 * being pinned by too many backends and each backend will only pin once
+	 * because of bufmgr.c's PrivateRefCount array.
+	 */
+	int32		pinning_backends;
 } BufferCachePagesRec;
 
 
@@ -61,6 +68,7 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 	MemoryContext oldcontext;
 	BufferCachePagesContext *fctx;		/* User function context. */
 	TupleDesc	tupledesc;
+	TupleDesc	expected_tupledesc;
 	HeapTuple	tuple;
 
 	if (SRF_IS_FIRSTCALL())
@@ -76,8 +84,23 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 		/* Create a user function context for cross-call persistence */
 		fctx = (BufferCachePagesContext *) palloc(sizeof(BufferCachePagesContext));
 
+		/*
+		 * To smoothly support upgrades from version 1.0 of this extension
+		 * transparently handle the (non-)existance of the pinning_backends
+		 * column. We unfortunately have to get the result type for that... -
+		 * we can't use the result type determined by the function definition
+		 * without potentially crashing when somebody uses the old (or even
+		 * wrong) function definition though.
+		 */
+		if (get_call_result_type(fcinfo, NULL, &expected_tupledesc) != TYPEFUNC_COMPOSITE)
+			elog(ERROR, "return type must be a row type");
+
+		if (expected_tupledesc->natts < NUM_BUFFERCACHE_PAGES_MIN_ELEM ||
+			expected_tupledesc->natts > NUM_BUFFERCACHE_PAGES_ELEM)
+			elog(ERROR, "incorrect number of output arguments");
+
 		/* Construct a tuple descriptor for the result rows. */
-		tupledesc = CreateTemplateTupleDesc(NUM_BUFFERCACHE_PAGES_ELEM, false);
+		tupledesc = CreateTemplateTupleDesc(expected_tupledesc->natts, false);
 		TupleDescInitEntry(tupledesc, (AttrNumber) 1, "bufferid",
 						   INT4OID, -1, 0);
 		TupleDescInitEntry(tupledesc, (AttrNumber) 2, "relfilenode",
@@ -95,6 +118,10 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 		TupleDescInitEntry(tupledesc, (AttrNumber) 8, "usage_count",
 						   INT2OID, -1, 0);
 
+		if (expected_tupledesc->natts == NUM_BUFFERCACHE_PAGES_ELEM)
+			TupleDescInitEntry(tupledesc, (AttrNumber) 9, "pinning_backends",
+							   INT4OID, -1, 0);
+
 		fctx->tupdesc = BlessTupleDesc(tupledesc);
 
 		/* Allocate NBuffers worth of BufferCachePagesRec records. */
@@ -110,17 +137,12 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 		MemoryContextSwitchTo(oldcontext);
 
 		/*
-		 * To get a consistent picture of the buffer state, we must lock all
-		 * partitions of the buffer map.  Needless to say, this is horrible
-		 * for concurrency.  Must grab locks in increasing order to avoid
-		 * possible deadlocks.
-		 */
-		for (i = 0; i < NUM_BUFFER_PARTITIONS; i++)
-			LWLockAcquire(BufMappingPartitionLockByIndex(i), LW_SHARED);
-
-		/*
-		 * Scan though all the buffers, saving the relevant fields in the
+		 * Scan through all the buffers, saving the relevant fields in the
 		 * fctx->record structure.
+		 *
+		 * We don't hold the partition locks, so we don't get a consistent
+		 * snapshot across all buffers, but we do grab the buffer header
+		 * locks, so the information of each buffer is self-consistent.
 		 */
 		for (i = 0, bufHdr = BufferDescriptors; i < NBuffers; i++, bufHdr++)
 		{
@@ -134,6 +156,7 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 			fctx->record[i].forknum = bufHdr->tag.forkNum;
 			fctx->record[i].blocknum = bufHdr->tag.blockNum;
 			fctx->record[i].usagecount = bufHdr->usage_count;
+			fctx->record[i].pinning_backends = bufHdr->refcount;
 
 			if (bufHdr->flags & BM_DIRTY)
 				fctx->record[i].isdirty = true;
@@ -148,16 +171,6 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 
 			UnlockBufHdr(bufHdr);
 		}
-
-		/*
-		 * And release locks.  We do this in reverse order for two reasons:
-		 * (1) Anyone else who needs more than one of the locks will be trying
-		 * to lock them in increasing order; we don't want to release the
-		 * other process until it can get all the locks it needs. (2) This
-		 * avoids O(N^2) behavior inside LWLockRelease.
-		 */
-		for (i = NUM_BUFFER_PARTITIONS; --i >= 0;)
-			LWLockRelease(BufMappingPartitionLockByIndex(i));
 	}
 
 	funcctx = SRF_PERCALL_SETUP();
@@ -188,6 +201,8 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 			nulls[5] = true;
 			nulls[6] = true;
 			nulls[7] = true;
+			/* unused for v1.0 callers, but the array is always long enough */
+			nulls[8] = true;
 		}
 		else
 		{
@@ -205,6 +220,9 @@ pg_buffercache_pages(PG_FUNCTION_ARGS)
 			nulls[6] = false;
 			values[7] = Int16GetDatum(fctx->record[i].usagecount);
 			nulls[7] = false;
+			/* unused for v1.0 callers, but the array is always long enough */
+			values[8] = Int32GetDatum(fctx->record[i].pinning_backends);
+			nulls[8] = false;
 		}
 
 		/* Build and return the tuple. */

contrib/pg_buffercache/sql/pg_buffercache.sql

@@ -0,0 +1,22 @@
+CREATE EXTENSION pg_buffercache;
+
+select count(*) = (select setting::bigint
+                   from pg_settings
+                   where name = 'shared_buffers')
+from pg_buffercache;
+
+SELECT count(*) = (select setting::bigint
+                   from pg_settings
+                   where name = 'shared_buffers') *
+                   (select count(*) from gp_segment_configuration where role='p')
+                   as buffers
+FROM gp_buffercache;
+
+-- Check that the functions / views can't be accessed by default.
+CREATE ROLE buffercache_test;
+SET ROLE buffercache_test;
+SELECT * FROM pg_buffercache;
+SELECT * FROM pg_buffercache_pages() AS p (wrong int);
+SELECT * FROM gp_buffercache;
+RESET ROLE;
+DROP ROLE buffercache_test;

doc/src/sgml/pgbuffercache.sgml

@@ -106,6 +106,13 @@
       <entry>Clock-sweep access count</entry>
      </row>
 
+     <row>
+      <entry><structfield>pinning_backends</structfield></entry>
+      <entry><type>integer</type></entry>
+      <entry></entry>
+      <entry>Number of backends pinning this buffer</entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>

gpMgmt/bin/gpcheckcat

@@ -3356,6 +3356,10 @@ TableMainColumn['pg_type_encoding'] = ['typid', 'pg_type']
 TableMainColumn['pg_window'] = ['winfnoid', 'pg_proc']
 TableMainColumn['gp_distribution_policy'] = ['localoid', 'pg_class']
 TableMainColumn['pg_description'] = ['objoid', 'pg_description']
+TableMainColumn['pg_shdescription'] = ['objoid', 'pg_shdescription']
+TableMainColumn['pg_seclabel'] = ['objoid', 'pg_seclabel']
+TableMainColumn['pg_shseclabel'] = ['objoid', 'pg_shseclabel']
+TableMainColumn['pg_seclabels'] = ['objoid', 'pg_seclabels']
 
 # Table with OID (special case), these OIDs are known to be inconsistent
 TableMainColumn['pg_attrdef'] = ['adrelid', 'pg_class']

gpMgmt/bin/gppylib/mainUtils.py

@@ -26,7 +26,7 @@
 
 from gppylib import gplog
 from gppylib.commands import gp, unix
-from gppylib.commands.base import ExecutionError
+from gppylib.commands.base import ExecutionError, Command
 from gppylib.system import configurationInterface, configurationImplGpdb, fileSystemInterface, \
     fileSystemImplOs, osInterface, osImplNative, faultProberInterface, faultProberImplGpdb
 from optparse import OptionGroup, OptionParser, SUPPRESS_HELP
@@ -83,6 +83,13 @@ def release(self):
             if self.PID == self.read_pid():
                 # remove the dir and PID file inside of it
                 shutil.rmtree(self.path)
+
+                # Eventhough we remove the directory, it is not guaranteed that the directory is removed
+                # at the disk level. So it is necessary to make this call to sync the changes at the disk level.
+                # Refer https://stackoverflow.com/questions/7127075/what-exactly-is-file-flush-doing for more context.
+                cmd = Command("run sync command", unix.findCmdInPath('sync'))
+                cmd.run()
+
         except EnvironmentError as e:
             if e.errno == errno.ENOENT:
                 pass
@@ -162,7 +169,11 @@ def acquire(self):
             return None
 
         # look for a lock file
-        self.pidfilepid = self.pidlockfile.read_pid()
+        try:
+            self.pidfilepid = self.pidlockfile.read_pid()
+        except ValueError:
+            shutil.rmtree(self.ppath)
+
         if self.pidfilepid is not None:
 
             # we found a lock file

gpMgmt/test/behave/mgmt_utils/environment.py

@@ -71,6 +71,12 @@ def before_feature(context, feature):
         dbconn.execSQL(context.conn, 'insert into t1 values(1, 2)')
         dbconn.execSQL(context.conn, 'insert into t2 values(1, 3)')
         dbconn.execSQL(context.conn, 'insert into t3 values(1, 4)')
+        # minirepro tests require statistical data about the contents of the database
+        # we should execute 'ANALYZE' to fill the pg_statistic catalog table.
+        dbconn.execSQL(context.conn, 'analyze t1')
+        dbconn.execSQL(context.conn, 'analyze t2')
+        dbconn.execSQL(context.conn, 'analyze t3')
+        dbconn.execSQL(context.conn, 'create or replace function select_one() returns integer as $$ select 1 $$ language sql')
         context.conn.commit()
 
     if 'gppkg' in feature.tags: