Cherry pick from GPDB (02/07/2022-02/28/2022)

gpMgmt/bin/gpload.py

@@ -2254,9 +2254,9 @@ def get_reuse_staging_table_query(self, encoding_conditions):
         return:
             sql(string)
         '''
-        sql = """SELECT oid::regclass
-                 FROM pg_class
-                 WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions)
+        sql = """SELECT oid::regclass \
+FROM pg_class \
+WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions)
 
         self.log(self.DEBUG, "query used to identify reusable temporary relations: %s" % sql)
         return sql
@@ -2273,7 +2273,8 @@ def get_table_oid(self, tableName):
                 pass
         return None
 
-    def get_ext_schematable(self, schemaName, tableName):
+
+    def get_schematable(self, schemaName, tableName):
         '''
         return formated table name
         '''
@@ -2448,7 +2449,7 @@ def create_external_table(self):
                               AND n.nspname !~ '^pg_toast'"""
                 result = self.db.query(sql).getresult()
                 if len(result) > 0:
-                    self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName)
+                    self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName)
                     self.log(self.INFO, "reusing external staging table %s" % self.extSchemaTable)
                     return
             # staging table is not specified, we need to find it manually
@@ -2468,7 +2469,7 @@ def create_external_table(self):
                     self.extTableName = (resultList[0])[0]
                     # fast match result is only table name, so we need add schema info
                     if self.fast_match:
-                        self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName)
+                        self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName)
                     else:
                         self.extSchemaTable = self.extTableName
                     self.log(self.INFO, "reusing external table %s" % self.extSchemaTable)
@@ -2479,13 +2480,13 @@ def create_external_table(self):
                 # around
 
                 self.extTableName = "ext_gpload_reusable_%s" % self.unique_suffix
-                self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_ext_schematable(self.extSchemaName, self.extTableName))
+                self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_schematable(self.extSchemaName, self.extTableName))
 
         # process the single quotes in order to successfully create an external table.
         self.formatOpts = self.formatOpts.replace("'\''","E'\\''")
 
         # construct a CREATE EXTERNAL TABLE statement and execute it
-        self.extSchemaTable = self.get_ext_schematable(self.extSchemaName, self.extTableName)
+        self.extSchemaTable = self.get_schematable(self.extSchemaName, self.extTableName)
         sql = "create external table %s" % self.extSchemaTable
         sql += "(%s)" % ','.join(['%s %s' % (a[0], a[1]) for a in from_cols])
 
@@ -2566,10 +2567,12 @@ def create_staging_table(self):
             # we no longer need the timestamp, since we will never want to create few
             # tables with same encoding_conditions
             self.staging_table_name = "staging_gpload_reusable_%s" % (encoding_conditions)
+            self.staging_table_name = self.get_schematable(self.extSchemaName, self.staging_table_name)
             self.log(self.INFO, "did not find a staging table to reuse. creating %s" % self.staging_table_name)
 
         # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table,
         # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True.
+
         sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name)
         cols = ['"%s" %s' % (a[0], a[1]) for a in target_columns]
         sql += "(%s)" % ','.join(cols)
@@ -2882,13 +2885,18 @@ def do_method(self):
         # Is the table to be truncated before the load?
         preload = self.getconfig('gpload:preload', list, default=None)
         method = self.getconfig('gpload:output:mode', str, 'insert').lower()
+        external = self.getconfig('gpload:external', list, default=None)
         self.log_errors = self.getconfig('gpload:input:log_errors', bool, False)
         truncate = False
         self.reuse_tables = False
 
         if not self.options.no_auto_trans and not method=='insert':
             self.db.query("BEGIN")
 
+        self.extSchemaName = self.getconfig('gpload:external:schema', str, None)
+        if self.extSchemaName == '%':
+            self.extSchemaName = self.schema
+
         if preload:
             truncate = self.getconfig('gpload:preload:truncate',bool,False)
             self.reuse_tables = self.getconfig('gpload:preload:reuse_tables',bool,False)

gpMgmt/bin/gpload_test/gpload2/query33.ans

@@ -1,7 +1,7 @@
 2018-07-24 06:14:29|INFO|gpload session started 2018-07-24 06:14:29
 2018-07-24 06:14:29|INFO|setting schema 'public' for table 'texttable'
 2018-07-24 06:14:29|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
-2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3
+2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3
 2018-07-24 06:14:29|INFO|did not find an external table to reuse. creating test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002
 2018-07-24 06:14:29|INFO|running time: 0.40 seconds
 2018-07-24 06:14:29|INFO|rows Inserted          = 16
@@ -11,7 +11,7 @@
 2018-07-24 06:14:30|INFO|gpload session started 2018-07-24 06:14:30
 2018-07-24 06:14:30|INFO|setting schema 'public' for table 'texttable'
 2018-07-24 06:14:30|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
-2018-07-24 06:14:30|INFO|reusing staging table staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3
+2018-07-24 06:14:30|INFO|reusing staging table test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3
 2018-07-24 06:14:30|INFO|reusing external table test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002
 2018-07-24 06:14:30|INFO|running time: 0.31 seconds
 2018-07-24 06:14:30|INFO|rows Inserted          = 0

gpMgmt/bin/gpstop

@@ -34,6 +34,7 @@ try:
     from gppylib.gp_era import GpEraFile
     from gppylib.operations.utils import ParallelOperation, RemoteOperation
     from gppylib.operations.rebalanceSegments import ReconfigDetectionSQLQueryCommand
+    from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts
 except ImportError as e:
     sys.exit('ERROR: Cannot import modules.  Please check that you have sourced greenplum_path.sh.  Detail: ' + str(e))
 
@@ -514,6 +515,10 @@ class GpStop:
         if self.gparray.standbyCoordinator:
             standby = self.gparray.standbyCoordinator
 
+            if get_unreachable_segment_hosts([standby.hostname], 1):
+                logger.warning("Standby is unreachable, skipping shutdown on standby")
+                return True
+
             logger.info("Stopping coordinator standby host %s mode=%s" % (standby.hostname, self.mode))
             try:
                 cmd = SegmentStop("stopping coordinator standby",

gpMgmt/test/behave/mgmt_utils/gpstop.feature

@@ -27,3 +27,13 @@ Feature: gpstop behave tests
           And gpstop should print "There were 1 user connections at the start of the shutdown" to stdout
           And gpstop should print "'\(s\)mart_mode', '\(f\)ast_mode', '\(i\)mmediate_mode'" to stdout
          Then gpstop should return a return code of 0
+
+    @demo_cluster
+    Scenario: gpstop succeeds even if the standby host is unreachable
+        Given the database is running
+          And the catalog has a standby coordinator entry
+         When the standby host is made unreachable
+          And the user runs "gpstop -a"
+         Then gpstop should print "Standby is unreachable, skipping shutdown on standby" to stdout
+          And gpstop should return a return code of 0
+          And the standby host is made reachable

gpMgmt/test/behave/mgmt_utils/steps/gpstart.py

@@ -37,30 +37,36 @@ def change_hostname(content, preferred_role, hostname):
 def impl(context):
     change_hostname(-1, 'm', 'invalid_host')
 
-    def cleanup(context):
-        """
-        Reverses the above SQL by starting up in coordinator-only utility mode. Since
-        the standby host is incorrect, a regular gpstart call won't work.
-        """
-        utils.stop_database_if_started(context)
-
-        subprocess.check_call(['gpstart', '-am'])
-        _run_sql("""
-            SET allow_system_table_mods='true';
-            UPDATE gp_segment_configuration
-               SET hostname = coordinator.hostname,
-                    address = coordinator.address
-              FROM (
-                     SELECT hostname, address
-                       FROM gp_segment_configuration
-                      WHERE content = -1 and role = 'p'
-                   ) coordinator
-             WHERE content = -1 AND role = 'm'
-        """, {'gp_role': 'utility'})
-        subprocess.check_call(['gpstop', '-am'])
-
     context.add_cleanup(cleanup, context)
 
+@when('the standby host is made reachable')
+@then('the standby host is made reachable')
+def impl(context):
+    cleanup(context)
+
+"""
+Reverses the changes done by change_hostname() function by starting up cluster in master-only utility mode. 
+Since the standby host is incorrect, a regular gpstart call won't work.
+"""
+def cleanup(context):
+
+    utils.stop_database_if_started(context)
+
+    subprocess.check_call(['gpstart', '-am'])
+    _run_sql("""
+        SET allow_system_table_mods='true';
+        UPDATE gp_segment_configuration
+           SET hostname = coordinator.hostname,
+                address = coordinator.address
+          FROM (
+                 SELECT hostname, address
+                   FROM gp_segment_configuration
+                  WHERE content = -1 and role = 'p'
+               ) coordinator
+         WHERE content = -1 AND role = 'm'
+    """, {'gp_role': 'utility'})
+    subprocess.check_call(['gpstop', '-am'])
+
 def _handle_sigpipe():
     """
     Work around https://bugs.python.org/issue1615376, which is not fixed until

src/backend/access/aocs/aocsam.c

@@ -507,13 +507,15 @@ aocs_beginscan(Relation relation,
 	RelationIncrementReferenceCount(relation);
 
 	/*
-	 * the append-only meta data should never be fetched with
+	 * The append-only meta data should never be fetched with
 	 * SnapshotAny as bogus results are returned.
+	 * We use SnapshotSelf for metadata, as regular MVCC snapshot can hide newly
+	 * globally inserted tuples from global index build process.
 	 */
 	if (snapshot != SnapshotAny)
 		aocsMetaDataSnapshot = snapshot;
 	else
-		aocsMetaDataSnapshot = GetTransactionSnapshot();
+		aocsMetaDataSnapshot = SnapshotSelf;
 
 	seginfo = GetAllAOCSFileSegInfo(relation, aocsMetaDataSnapshot, &total_seg, NULL);
 	return aocs_beginscan_internal(relation,