Merge pull request #290 from arenadata/5.29.2-sync

ADBDEV-2361 5.29.2 sync

config/orca.m4

@@ -40,10 +40,10 @@ AC_RUN_IFELSE([AC_LANG_PROGRAM([[
 #include <string.h>
 ]],
 [
-return strncmp("3.120.", GPORCA_VERSION_STRING, 6);
+return strncmp("3.121.", GPORCA_VERSION_STRING, 6);
 ])],
 [AC_MSG_RESULT([[ok]])],
-[AC_MSG_ERROR([Your ORCA version is expected to be 3.120.XXX])]
+[AC_MSG_ERROR([Your ORCA version is expected to be 3.121.XXX])]
 )
 AC_LANG_POP([C++])
 ])# PGAC_CHECK_ORCA_VERSION

configure

@@ -582,8 +582,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='Greenplum Database'
 PACKAGE_TARNAME='greenplum-database'
-PACKAGE_VERSION='5.29.1'
-PACKAGE_STRING='Greenplum Database 5.29.1'
+PACKAGE_VERSION='5.29.2'
+PACKAGE_STRING='Greenplum Database 5.29.2'
 PACKAGE_BUGREPORT='[email protected]'
 PACKAGE_URL=''
 
@@ -12570,7 +12570,7 @@ int
 main ()
 {
 
-return strncmp("3.120.", GPORCA_VERSION_STRING, 6);
+return strncmp("3.121.", GPORCA_VERSION_STRING, 6);
 
   ;
   return 0;
@@ -12580,7 +12580,7 @@ if ac_fn_cxx_try_run "$LINENO"; then :
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5
 $as_echo "ok" >&6; }
 else
-  as_fn_error $? "Your ORCA version is expected to be 3.120.XXX" "$LINENO" 5
+  as_fn_error $? "Your ORCA version is expected to be 3.121.XXX" "$LINENO" 5
 
 fi
 rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \

gpAux/releng/releng.mk

@@ -120,7 +120,7 @@ sync_tools: opt_write_test /opt/releng/apache-ant
 	-Divyrepo.user=$(IVYREPO_USER) -Divyrepo.passwd="$(IVYREPO_PASSWD)" -quiet resolve);
 
 ifeq "$(findstring aix,$(BLD_ARCH))" ""
-	LD_LIBRARY_PATH='' wget --no-check-certificate -q -O - https://github.com/greenplum-db/gporca/releases/download/v3.120.0/bin_orca_centos5_release.tar.gz | tar zxf - -C $(BLD_TOP)/ext/$(BLD_ARCH)
+	LD_LIBRARY_PATH='' wget --no-check-certificate -q -O - https://github.com/greenplum-db/gporca/releases/download/v3.121.0/bin_orca_centos5_release.tar.gz | tar zxf - -C $(BLD_TOP)/ext/$(BLD_ARCH)
 endif
 
 clean_tools: opt_write_test

gpMgmt/bin/gpload.py

@@ -1920,29 +1920,43 @@ def check_enable_custom_format(self):
                 self.enable_custom_format = 1
 
     def check_custom_formatter(self):
+        # Check if load dataflow extension is necessary
+        load_dataflow = False
+
+        # sql pre processing configured guc and used text
+        sql = self.getconfig('gpload:sql', list, default=None)
+        before = None
+        if sql:
+            before = self.getconfig('gpload:sql:before', unicode, default=None)
+            if before:
+                if 'dataflow.prefer_custom_text' in before.lower().replace(" ", ""):
+                    load_dataflow = True
+
         # Check if 'text_in' custom formatter can be used
         self.support_cusfmt = 0
-        try:
-            # make sure dataflow extension has been created.
-            queryString = """CREATE EXTENSION IF NOT EXISTS dataflow;"""
-            self.db.query(queryString.encode('utf-8'))
-            # load gpss.so to enable "dataflow.prefer_custom_text" guc.
-            queryString = """SELECT dataflow_version();"""
-            self.db.query(queryString.encode('utf-8'))
-            # show "dataflow.prefer_custom_text" guc, this guc only exists in gpdb6.
-            queryString = """SHOW dataflow.prefer_custom_text;"""
-            self.db.query(queryString.encode('utf-8'))
-
-            queryString = """SELECT c.oid FROM pg_catalog.pg_proc c 
-                               LEFT JOIN pg_catalog.pg_namespace n
-                               ON n.oid = c.pronamespace
-                               WHERE c.proname = 'text_in';"""
-            resultList = self.db.query(queryString.encode('utf-8')).getresult()
-            if len(resultList) > 0:
-                self.support_cusfmt = 1
 
-        except Exception, e:
-            self.log(self.DEBUG, 'could not run SQL "%s": %s' % (queryString, unicode(e)))
+        if load_dataflow:
+            try:
+                # make sure dataflow extension has been created.
+                queryString = """CREATE EXTENSION IF NOT EXISTS dataflow;"""
+                self.db.query(queryString.encode('utf-8'))
+                # load gpss.so to enable "dataflow.prefer_custom_text" guc.
+                queryString = """SELECT dataflow_version();"""
+                self.db.query(queryString.encode('utf-8'))
+                # show "dataflow.prefer_custom_text" guc, this guc only exists in gpdb6.
+                queryString = """SHOW dataflow.prefer_custom_text;"""
+                self.db.query(queryString.encode('utf-8'))
+
+                queryString = """SELECT c.oid FROM pg_catalog.pg_proc c 
+                                   LEFT JOIN pg_catalog.pg_namespace n
+                                   ON n.oid = c.pronamespace
+                                   WHERE c.proname = 'text_in';"""
+                resultList = self.db.query(queryString.encode('utf-8')).getresult()
+                if len(resultList) > 0:
+                    self.support_cusfmt = 1
+
+            except Exception, e:
+                self.log(self.DEBUG, 'could not run SQL "%s": %s' % (queryString, unicode(e)))
 
     def read_table_metadata(self):
         # KAS Note to self. If schema is specified, then probably should use PostgreSQL rules for defining it.
@@ -2980,7 +2994,9 @@ def run2(self):
         start = time.time()
         self.read_config()
         self.setup_connection()
-        self.check_custom_formatter()
+        # Custom formatter only works for gpdb 6 and abover
+        if noGpVersion == False and self.gpdb_version >= "6.0.0":
+            self.check_custom_formatter()
         self.read_table_metadata()
         self.read_columns()
         self.read_mapping()

src/backend/cdb/cdbresynchronizechangetracking.c

@@ -2319,7 +2319,10 @@ ChangeTracking_MarkFullResyncLockAcquired(void)
 	FileRep_SetSegmentState(SegmentStateChangeTrackingDisabled, FaultTypeNotInitialized);
 
 	getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
-	Assert(segmentState == SegmentStateChangeTrackingDisabled);
+	Assert(segmentState == SegmentStateChangeTrackingDisabled /* common scenario */ ||
+		(fileRepRole == FileRepNoRoleConfigured &&
+		 segmentState == SegmentStateNotInitialized &&
+		 dataState == DataStateNotInitialized) /* PMModeMirrorlessSegment doing gpaddmirrors */);
 }
 
 /*

src/backend/libpq/pqcomm.c

@@ -1166,6 +1166,18 @@ pq_getstring(StringInfo s)
 	}
 }
 
+/* --------------------------------
+ *		pq_buffer_has_data		- is any buffered data available to read?
+ *
+ * This will *not* attempt to read more data.
+ * --------------------------------
+ */
+bool
+pq_buffer_has_data(void)
+{
+	return (PqRecvPointer < PqRecvLength);
+}
+
 
 /* --------------------------------
  *		pq_getmessage	- get a message with length word from connection

src/backend/postmaster/postmaster.c

@@ -2738,6 +2738,19 @@ ProcessStartupPacket(Port *port, bool SSLdone)
 		if (SSLok == 'S' && secure_open_server(port) == -1)
 			return STATUS_ERROR;
 #endif
+
+		/*
+		 * At this point we should have no data already buffered.  If we do,
+		 * it was received before we performed the SSL handshake, so it wasn't
+		 * encrypted and indeed may have been injected by a man-in-the-middle.
+		 * We report this case to the client.
+		 */
+		if (pq_buffer_has_data())
+			ereport(FATAL,
+					(errcode(ERRCODE_PROTOCOL_VIOLATION),
+					 errmsg("received unencrypted data after SSL request"),
+					 errdetail("This could be either a client-software bug or evidence of an attempted man-in-the-middle attack.")));
+
 		/* regular startup packet, cancel, etc packet should follow... */
 		/* but not another SSL negotiation request */
 		return ProcessStartupPacket(port, true);

src/backend/storage/lmgr/proc.c

@@ -1448,15 +1448,22 @@ CheckDeadLock(void)
 		if (Gp_role == GP_ROLE_DISPATCH && IsResQueueEnabled() &&
 			LOCK_LOCKMETHOD(*(MyProc->waitLock)) == RESOURCE_LOCKMETHOD)
 		{
-			ResRemoveFromWaitQueue(MyProc, 
-								   LockTagHashCode(&(MyProc->waitLock->tag)));
 			/*
-			 * lockAwaited's lock/proclock pointers are dangling after the call
-			 * to ResRemoveFromWaitQueue(). So clean up the locallock as well,
-			 * to avoid de-referencing them in the eventual ResLockRelease() in
-			 * ResLockPortal/ResLockUtilityPortal.
+			 * If there are no other locked portals resident in this backend
+			 * (i.e. nLocks == 0), lockAwaited's lock/proclock pointers are dangling
+			 * after the following call to ResRemoveFromWaitQueue(). So clean up the
+			 * locallock as well, to avoid de-referencing them in the eventual
+			 * ResLockRelease() in ResLockPortal()/ResLockUtilityPortal().
+			 *
+			 * If there are other locked portals resident in this backend
+			 * (i.e. nLocks > 0), as always, the lock and proclock cannot be cleaned
+			 * up now. Thus, defer the cleanup of the locallock.
 			 */
-			RemoveLocalLock(lockAwaited);
+			if (MyProc->waitProcLock->nLocks == 0)
+				RemoveLocalLock(lockAwaited);
+
+			ResRemoveFromWaitQueue(MyProc,
+								   LockTagHashCode(&(MyProc->waitLock->tag)));
 		}
 		else
 		{
@@ -1997,14 +2004,21 @@ ResLockWaitCancel(void)
 			/* We should only be trying to cancel resource locks. */
 			Assert(LOCALLOCK_LOCKMETHOD(*lockAwaited) == RESOURCE_LOCKMETHOD);
 
-			ResRemoveFromWaitQueue(MyProc, lockAwaited->hashcode);
 			/*
-			 * lockAwaited's lock/proclock pointers are dangling after the call
-			 * to ResRemoveFromWaitQueue(). So clean up the locallock as well,
-			 * to avoid de-referencing them in the eventual ResLockRelease() in
-			 * ResLockPortal/ResLockUtilityPortal.
+			 * If there are no other locked portals resident in this backend
+			 * (i.e. nLocks == 0), lockAwaited's lock/proclock pointers are dangling
+			 * after the following call to ResRemoveFromWaitQueue(). So clean up the
+			 * locallock as well, to avoid de-referencing them in the eventual
+			 * ResLockRelease() in ResLockPortal()/ResLockUtilityPortal().
+			 *
+			 * If there are other locked portals resident in this backend
+			 * (i.e. nLocks > 0), as always, the lock and proclock cannot be cleaned
+			 * up now. Thus, defer the cleanup of the locallock.
 			 */
-			RemoveLocalLock(lockAwaited);
+			if (MyProc->waitProcLock->nLocks == 0)
+				RemoveLocalLock(lockAwaited);
+
+			ResRemoveFromWaitQueue(MyProc, lockAwaited->hashcode);
 		}
 
 		lockAwaited = NULL;

src/backend/utils/resscheduler/resqueue.c

@@ -323,7 +323,11 @@ ResLockAcquire(LOCKTAG *locktag, ResPortalIncrement *incrementSet)
 		lock->requested[lockmode]--;
 		Assert((lock->nRequested >= 0) && (lock->requested[lockmode] >= 0));
 
-		/* Clean up this lock. */
+		/*
+		 * Clean up the locallock. Since a single locallock can represent
+		 * multiple locked portals in the same backend, we can only remove it if
+		 * this is the last portal.
+		 */
 		if (proclock->nLocks == 0)
 			RemoveLocalLock(locallock);
 
@@ -385,7 +389,11 @@ ResLockAcquire(LOCKTAG *locktag, ResPortalIncrement *incrementSet)
 		lock->requested[lockmode]--;
 		Assert((lock->nRequested >= 0) && (lock->requested[lockmode] >= 0));
 
-		/* Clean up this lock. */
+		/*
+		 * Clean up the locallock. Since a single locallock can represent
+		 * multiple locked portals in the same backend, we can only remove it if
+		 * this is the last portal.
+		 */
 		if (proclock->nLocks == 0)
 			RemoveLocalLock(locallock);
 
@@ -597,6 +605,11 @@ ResLockRelease(LOCKTAG *locktag, uint32 resPortalId)
 	if (!incrementSet)
 	{
 		elog(DEBUG1, "Resource queue %d: increment not found on unlock", locktag->locktag_field1);
+		/*
+		 * Clean up the locallock. Since a single locallock can represent
+		 * multiple locked portals in the same backend, we can only remove it if
+		 * this is the last portal.
+		 */
 		if (proclock->nLocks == 0)
 		{
 			RemoveLocalLock(locallock);
@@ -616,6 +629,10 @@ ResLockRelease(LOCKTAG *locktag, uint32 resPortalId)
 
 	/*
 	 * Perform clean-up, waking up any waiters!
+	 *
+	 * Clean up the locallock. Since a single locallock can represent
+	 * multiple locked portals in the same backend, we can only remove it if
+	 * this is the last portal.
 	 */
 	if (proclock->nLocks == 0)
 		RemoveLocalLock(locallock);

src/include/libpq/libpq.h

@@ -59,6 +59,7 @@ extern int	pq_getmessage(StringInfo s, int maxlen);
 extern int	pq_getbyte(void);
 extern int	pq_peekbyte(void);
 extern int	pq_getbyte_if_available(unsigned char *c);
+extern bool pq_buffer_has_data(void);
 extern int	pq_putbytes(const char *s, size_t len);
 extern int	pq_flush(void);
 extern int	pq_flush_if_writable(void);

src/interfaces/libpq/fe-connect.c

@@ -2225,6 +2225,19 @@ PQconnectPoll(PGconn *conn)
 				pollres = pqsecure_open_client(conn);
 				if (pollres == PGRES_POLLING_OK)
 				{
+					/*
+					 * At this point we should have no data already buffered.
+					 * If we do, it was received before we performed the SSL
+					 * handshake, so it wasn't encrypted and indeed may have
+					 * been injected by a man-in-the-middle.
+					 */
+					if (conn->inCursor != conn->inEnd)
+					{
+						appendPQExpBufferStr(&conn->errorMessage,
+											 libpq_gettext("received unencrypted data after SSL response\n"));
+						goto error_return;
+					}
+
 					/* SSL handshake done, ready to send startup packet */
 					conn->status = CONNECTION_MADE;
 					return PGRES_POLLING_WRITING;

src/test/isolation2/expected/resource_queue.out

@@ -39,6 +39,13 @@ END
 2:END;
 END
 
+-- Sanity check: Ensure that the resource queue is now empty.
+0: SELECT rsqcountlimit, rsqcountvalue from pg_resqueue_status WHERE rsqname = 'rq_concurrency_test';
+rsqcountlimit|rsqcountvalue
+-------------+-------------
+1            |0            
+(1 row)
+
 0:DROP role role_concurrency_test;
 DROP
 0:DROP RESOURCE QUEUE rq_concurrency_test;

src/test/isolation2/expected/resource_queue_cancel.out

@@ -0,0 +1,57 @@
+-- Simple test for cancellation of a query stuck on a resource queue when the
+-- active statements limit is reached.
+
+0:CREATE RESOURCE QUEUE rq_cancel WITH (active_statements = 1);
+CREATE
+0:CREATE ROLE role_cancel RESOURCE QUEUE rq_cancel;
+CREATE
+
+-- Consume an active statement in session 1.
+1:SET ROLE role_cancel;
+SET
+1:BEGIN;
+BEGIN
+1:DECLARE c CURSOR FOR SELECT 0;
+DECLARE
+
+-- Make session 2 wait on the resource queue lock.
+2:SET ROLE role_cancel;
+SET
+2&:SELECT 100;  <waiting ...>
+
+-- Cancel SELECT from session 2.
+0:SELECT pg_cancel_backend(procpid) FROM pg_stat_activity WHERE current_query='SELECT 100;';
+pg_cancel_backend
+-----------------
+t                
+(1 row)
+
+-- Now once we end session 1's transaction, we should be able to consume the
+-- vacated active statement slot in session 2.
+1:END;
+END
+
+2<:  <... completed>
+ERROR:  canceling statement due to user request
+2:END;
+END
+2:BEGIN;
+BEGIN
+2:DECLARE c CURSOR FOR SELECT 0;
+DECLARE
+
+2:END;
+END
+
+-- Sanity check: Ensure that the resource queue is now empty.
+0:SELECT rsqcountlimit, rsqcountvalue FROM pg_resqueue_status WHERE rsqname = 'rq_cancel';
+rsqcountlimit|rsqcountvalue
+-------------+-------------
+1            |0            
+(1 row)
+
+-- Cleanup
+0:DROP ROLE role_cancel;
+DROP
+0:DROP RESOURCE QUEUE rq_cancel;
+DROP