diff --git a/gpMgmt/bin/analyzedb b/gpMgmt/bin/analyzedb index a2ad49a7ff3..2a0a0ce81d7 100755 --- a/gpMgmt/bin/analyzedb +++ b/gpMgmt/bin/analyzedb @@ -25,16 +25,15 @@ from contextlib import closing import pipes # for shell-quoting, pipes.quote() import fcntl import itertools - +import psycopg2 try: - import pg - from gppylib import gplog, pgconf, userinput from gppylib.commands.base import Command, WorkerPool, Worker from gppylib.operations import Operation from gppylib.gpversion import GpVersion from gppylib.db import dbconn from gppylib.operations.unix import CheckDir, CheckFile, MakeDir + from gppylib.utils import escape_string except ImportError as e: sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -166,7 +165,7 @@ def validate_schema_exists(pg_port, dbname, schema): try: dburl = dbconn.DbURL(port=pg_port, dbname=dbname) conn = dbconn.connect(dburl) - count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % pg.escape_string(schema)) + count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % escape_string(schema)) if count == 0: raise ExceptionNoStackTraceNeeded("Schema %s does not exist in database %s." % (schema, dbname)) finally: @@ -213,7 +212,7 @@ def get_partition_state_tuples(pg_port, dbname, catalog_schema, partition_info): try: modcount_sql = "select to_char(coalesce(sum(modcount::bigint), 0), '999999999999999999999') from gp_dist_random('%s.%s')" % (catalog_schema, tupletable) modcount = dbconn.querySingleton(conn, modcount_sql) - except pg.DatabaseError as e: + except psycopg2.DatabaseError as e: if "does not exist" in str(e): logger.info("Table %s.%s (%s) no longer exists and will not be analyzed", schemaname, partition_name, tupletable) else: @@ -971,7 +970,7 @@ def get_oid_str(table_list): def regclass_schema_tbl(schema, tbl): schema_tbl = "%s.%s" % (escape_identifier(schema), escape_identifier(tbl)) - return "to_regclass('%s')" % (pg.escape_string(schema_tbl)) + return "to_regclass('%s')" % (escape_string(schema_tbl)) # Escape double-quotes in a string, so that the resulting string is suitable for @@ -1239,7 +1238,7 @@ def validate_tables(conn, tablenames): while curr_batch < nbatches: batch = tablenames[curr_batch * batch_size:(curr_batch + 1) * batch_size] - oid_str = ','.join(map((lambda x: "('%s')" % pg.escape_string(x)), batch)) + oid_str = ','.join(map((lambda x: "('%s')" % escape_string(x)), batch)) if not oid_str: break @@ -1255,7 +1254,7 @@ def get_include_cols_from_exclude(conn, schema, table, exclude_cols): """ Given a list of excluded columns of a table, get the list of included columns """ - quoted_exclude_cols = ','.join(["'%s'" % pg.escape_string(x) for x in exclude_cols]) + quoted_exclude_cols = ','.join(["'%s'" % escape_string(x) for x in exclude_cols]) oid_str = regclass_schema_tbl(schema, table) cols = run_sql(conn, GET_INCLUDED_COLUMNS_FROM_EXCLUDE_SQL % (oid_str, quoted_exclude_cols)) @@ -1271,7 +1270,7 @@ def validate_columns(conn, schema, table, column_list): return sql = VALIDATE_COLUMN_NAMES_SQL % (regclass_schema_tbl(schema, table), - ','.join(["'%s'" % pg.escape_string(x) for x in column_list])) + ','.join(["'%s'" % escape_string(x) for x in column_list])) valid_col_count = dbconn.querySingleton(conn, sql) if int(valid_col_count) != len(column_list): diff --git a/gpMgmt/bin/gpactivatestandby b/gpMgmt/bin/gpactivatestandby index 813d6cd3f75..fa8dd32e505 100755 --- a/gpMgmt/bin/gpactivatestandby +++ b/gpMgmt/bin/gpactivatestandby @@ -21,10 +21,9 @@ import time import shutil import tempfile from datetime import datetime, timedelta - +import psycopg2 # import GPDB modules try: - import pg as pygresql from gppylib.commands import unix, gp, pg from gppylib.db import dbconn from gppylib.gpparseopts import OptParser, OptChecker, OptionGroup, SUPPRESS_HELP @@ -332,7 +331,7 @@ def promote_standby(coordinator_data_dir): logger.info('Standby coordinator is promoted') conn.close() return True - except pygresql.InternalError as e: + except (psycopg2.InternalError, psycopg2.OperationalError) as e: pass time.sleep(1) diff --git a/gpMgmt/bin/gpcheckcat b/gpMgmt/bin/gpcheckcat index 4150acbae4e..bc019263f94 100755 --- a/gpMgmt/bin/gpcheckcat +++ b/gpMgmt/bin/gpcheckcat @@ -27,23 +27,20 @@ import re import sys import time from functools import reduce - +import psycopg2 +from psycopg2 import extras +from contextlib import closing try: from gppylib import gplog - from gppylib.db import dbconn from gppylib.gpcatalog import * from gppylib.commands.unix import * from gppylib.commands.gp import conflict_with_gpexpand from gppylib.system.info import * - from pgdb import DatabaseError from gpcheckcat_modules.unique_index_violation_check import UniqueIndexViolationCheck from gpcheckcat_modules.leaked_schema_dropper import LeakedSchemaDropper from gpcheckcat_modules.repair import Repair from gpcheckcat_modules.foreign_key_check import ForeignKeyCheck from gpcheckcat_modules.orphaned_toast_tables_check import OrphanedToastTablesCheck - - import pg - except ImportError as e: sys.exit('Error: unable to import module: ' + str(e)) @@ -133,7 +130,7 @@ class Global(): self.dbname = None self.firstdb = None self.alldb = [] - self.db = {} + self.conn = {} self.tmpdir = None self.reset_stmt_queues() @@ -201,30 +198,27 @@ def usage(exitarg=None): ############################### def getversion(): - db = connect() - curs = db.query(''' - select regexp_replace(version(), - E'.*PostgreSQL [^ ]+ .Cloudberry Database ([1-9]+.[0-9]+|main).*', - E'\\\\1') as ver;''') - - row = curs.getresult()[0] - version = row[0] - - logger.debug('got version %s' % version) - return version - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(''' + select regexp_replace(version(), + E'.*PostgreSQL [^ ]+ .Greenplum Database ([1-9]+.[0-9]+|main).*', + E'\\\\1') as ver;''') + row = curs.fetchone() + version = row[0] + logger.debug('got version %s' % version) + return version ############################### def getalldbs(): """ get all connectable databases """ - db = connect() - curs = db.query(''' - select datname from pg_database where datallowconn order by datname ''') - row = curs.getresult() - return row - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute('''select datname from pg_database where datallowconn order by datname''') + row = curs.fetchall() + return row ############################### def parseCommandLine(): @@ -333,19 +327,21 @@ def connect(user=None, password=None, host=None, port=None, try: logger.debug('connecting to %s:%s %s' % (host, port, database)) - db = pg.connect(host=host, port=port, user=user, - passwd=password, dbname=database, opt=options) - - except pg.InternalError as ex: + conn = psycopg2.connect(host=host, port=port, user=user, + password=password, dbname=database, options=options) + ## Don't execute query in a transaction block. + conn.set_session(autocommit=True) + except (psycopg2.InternalError, psycopg2.OperationalError) as ex: logger.fatal('could not connect to %s: "%s"' % (database, str(ex).strip())) exit(1) logger.debug('connected with %s:%s %s' % (host, port, database)) - return db + return conn -############# +# NOTE: We cannot use connect2() with contextmanager, since we manage the connection +# ourselves. def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): host = cfgrec['address'] port = cfgrec['port'] @@ -357,22 +353,22 @@ def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): key = "%s.%s.%s.%s.%s.%s.%s" % (host, port, datadir, user, password, database, str(utilityMode)) - conns = GV.db.get(key) + conns = GV.conn.get(key) if conns: return conns[0] conn = connect(host=host, port=port, user=user, password=password, database=database, utilityMode=utilityMode) if conn: - GV.db[key] = [conn, cfgrec] + GV.conn[key] = [conn, cfgrec] return conn class execThread(Thread): - def __init__(self, cfg, db, qry): + def __init__(self, cfg, conn, qry): self.cfg = cfg - self.db = db + self.conn = conn self.qry = qry self.curs = None self.error = None @@ -380,11 +376,11 @@ class execThread(Thread): def run(self): try: - self.curs = self.db.query(self.qry) + self.curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self.curs.execute(self.qry) except BaseException as e: self.error = e - def processThread(threads): batch = [] for th in threads: @@ -413,9 +409,9 @@ def connect2run(qry, col=None): # parallelise queries for dbid in GV.cfg: c = GV.cfg[dbid] - db = connect2(c) + conn = connect2(c) - thread = execThread(c, db, qry) + thread = execThread(c, conn, qry) thread.start() logger.debug('launching query thread %s for dbid %i' % (thread.getName(), dbid)) @@ -435,8 +431,8 @@ def connect2run(qry, col=None): err = [] for [cfg, curs] in batch: if col is None: - col = curs.listfields() - for row in curs.dictresult(): + col = [desc[0] for desc in curs.description] + for row in curs.fetchall(): err.append([cfg, col, row]) return err @@ -454,7 +450,6 @@ def formatErr(c, col, row): ############# def getGPConfiguration(): cfg = {} - db = connect() # note that in 4.0, sql commands cannot be run against the segment mirrors directly # so we filter out non-primary segment databases in the query qry = ''' @@ -464,13 +459,14 @@ def getGPConfiguration(): FROM gp_segment_configuration WHERE (role = 'p' or content < 0 ) ''' - curs = db.query(qry) - for row in curs.dictresult(): - if row['content'] == -1 and not row['isprimary']: - continue # skip standby coordinator - cfg[row['dbid']] = row - db.close() - return cfg + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + if row['content'] == -1 and not row['isprimary']: + continue # skip standby coordinator + cfg[row['dbid']] = row + return cfg def checkDistribPolicy(): logger.info('-----------------------------------') @@ -484,27 +480,27 @@ def checkDistribPolicy(): where pk.contype in('p', 'u') and d.policytype = 'p' and d.distkey = '' ''' - db = connect2(GV.cfg[GV.coordinator_dbid]) try: - curs = db.query(qry) - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] randomly distributed tables') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] randomly distributed tables') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: - logger.error(formatErr(e[0], e[1], e[2])) - for e in err: - cons = e[2] - removeIndexConstraint(cons['nspname'], cons['relname'], - cons['constraint']) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] randomly distributed tables') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] randomly distributed tables') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) + for e in err: + logger.error(formatErr(e[0], e[1], e[2])) + cons = e[2] + removeIndexConstraint(cons['nspname'], cons['relname'], + cons['constraint']) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkDistribPolicy') @@ -526,22 +522,23 @@ def checkDistribPolicy(): and not d.distkey::int2[] operator(pg_catalog.<@) pk.conkey ''' try: - curs = db.query(qry) - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] unique constraints') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] unique constraints') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: logger.error(formatErr(e[0], e[1], e[2])) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + err = [] + curs.execute(qry) + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] unique constraints') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] unique constraints') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) for e in err: + logger.error(formatErr(e[0], e[1], e[2])) cons = e[2] removeIndexConstraint(cons['nspname'], cons['relname'], cons['constraint']) @@ -557,7 +554,6 @@ def checkPartitionIntegrity(): logger.info('-----------------------------------') logger.info('Checking pg_partition ...') err = [] - db = connect() # MPP-11120: check for child partitions with different # distribution policies. @@ -571,74 +567,74 @@ def checkPartitionIntegrity(): and not (inhrelid in (select ftrelid from pg_catalog.pg_foreign_table) and pg_get_table_distributedby(inhrelid) = ''); ''' try: - curs = db.query(qry) - cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') - col_names = { - 'inhparent': 'table', - 'inhrelid': 'affected child', - 'dby_parent': 'table distribution key', - 'dby_child': 'child distribution key', - } - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], cols, row]) - - if not err: - logger.info('[OK] partition distribution policy check') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] partition distribution policy check') - logger.error('partition distribution policy check found %d issue(s)' % len(err)) - if len(err) > 100: - logger.error(qry) - - myprint( - '[ERROR]: child partition(s) are distributed differently from ' - 'the root partition, and must be manually redistributed, for ' - 'some tables. Check the gpcheckcat log for details.' - ) - logger.error('The following tables must be manually redistributed:') - - count = 0 - for e in err: - cfg = e[0] - col = e[1] - row = e[2] - - # TODO: generate a repair script for this row. This is - # difficult, since we can't redistribute child partitions - # directly. - - # report at most 100 rows, for brevity - if count == 100: - logger.error("...") - count += 1 - if count > 100: - continue - - if count == 0: - logger.error("--------") - logger.error(" " + " | ".join(map(col_names.get, col))) - logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) - - logger.error(" " + " | ".join([str(row[x]) for x in col])) - count += 1 - - logger.error( - 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' - 'the desired distribution key, on the partition root for each ' - 'affected table.' - ) + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') + col_names = { + 'inhparent': 'table', + 'inhrelid': 'affected child', + 'dby_parent': 'table distribution key', + 'dby_child': 'child distribution key', + } + + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], cols, row]) + + if not err: + logger.info('[OK] partition distribution policy check') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] partition distribution policy check') + logger.error('partition distribution policy check found %d issue(s)' % len(err)) + if len(err) > 100: + logger.error(qry) + + myprint( + '[ERROR]: child partition(s) are distributed differently from ' + 'the root partition, and must be manually redistributed, for ' + 'some tables. Check the gpcheckcat log for details.' + ) + logger.error('The following tables must be manually redistributed:') + + count = 0 + for e in err: + cfg = e[0] + col = e[1] + row = e[2] + + # TODO: generate a repair script for this row. This is + # difficult, since we can't redistribute child partitions + # directly. + + # report at most 100 rows, for brevity + if count == 100: + logger.error("...") + count += 1 + if count > 100: + continue + + if count == 0: + logger.error("--------") + logger.error(" " + " | ".join(map(col_names.get, col))) + logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) + + logger.error(" " + " | ".join([str(row[x]) for x in col])) + count += 1 + + logger.error( + 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' + 'the desired distribution key, on the partition root for each ' + 'affected table.' + ) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkPartitionIntegrity') myprint(' Execution error: ' + str(e)) - db.close() - checkPoliciesRepair() ############# @@ -705,7 +701,7 @@ Produce repair scripts to remove dangling entries of gp_fastsequence: ''' -def removeFastSequence(db): +def removeFastSequence(conn): ''' MPP-14758: gp_fastsequence does not get cleanup after a failed transaction (AO/CO) Note: this is slightly different from the normal foreign key check @@ -731,14 +727,15 @@ def removeFastSequence(db): ON r.gp_segment_id = cfg.content WHERE cfg.role = 'p'; """ - curs = db.query(qry) - for row in curs.dictresult(): - seg = row['dbid'] # dbid of targeted segment - name = 'gp_fastsequence tuple' # for comment purposes - table = 'gp_fastsequence' # table name - cols = {'objid': row['objid']} # column name and value - objname = 'gp_fastsequence' # for comment purposes - buildRemove(seg, name, table, cols, objname) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + seg = row['dbid'] # dbid of targeted segment + name = 'gp_fastsequence tuple' # for comment purposes + table = 'gp_fastsequence' # table name + cols = {'objid': row['objid']} # column name and value + objname = 'gp_fastsequence' # for comment purposes + buildRemove(seg, name, table, cols, objname) except Exception as e: logger.error('removeFastSequence: ' + str(e)) @@ -800,42 +797,41 @@ def drop_leaked_schemas(leaked_schema_dropper, dbname): logger.info('-----------------------------------') logger.info('Checking for leaked temporary schemas') - db_connection = connect(database=dbname) try: - dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(db_connection) - if not dropped_schemas: - logger.info('[OK] temporary schemas') - else: - logger.info('[FAIL] temporary schemas') - myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) - logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) + with closing(connect(database=dbname)) as conn: + dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(conn) + if not dropped_schemas: + logger.info('[OK] temporary schemas') + else: + logger.info('[FAIL] temporary schemas') + myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) + logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) - finally: - db_connection.close() def checkDepend(): # Check for dependencies on non-existent objects logger.info('-----------------------------------') logger.info('Checking Object Dependencies') - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: - # Catalogs that link up to pg_depend/pg_shdepend - qry = """ - select relname from pg_class c - where relkind='r' - and relnamespace=%d - and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) - """ % PG_CATALOG_OID - curs = db.query(qry) - catalogs = [] - for row in curs.getresult(): - catalogs.append(row[0]) - - checkDependJoinCatalog(catalogs) - checkCatalogJoinDepend(catalogs) + # Catalogs that link up to pg_depend/pg_shdepend + qry = """ + select relname from pg_class c + where relkind='r' + and relnamespace=%d + and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) + """ % PG_CATALOG_OID + curs.execute(qry) + catalogs = [] + for row in curs.fetchall(): + catalogs.append(row[0]) + + checkDependJoinCatalog(catalogs) + checkCatalogJoinDepend(catalogs) def checkDependJoinCatalog(catalogs): # Construct subquery that will verify that all (classid, objid) @@ -987,7 +983,6 @@ def checkOwners(): # # - Between 3.3 and 4.0 the ao segment columns migrated from pg_class # to pg_appendonly. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, coalesce(o.relname, c.relname) as relname, a.rolname, m.rolname as coordinator_rolname @@ -1004,20 +999,21 @@ def checkOwners(): where c.relowner <> r.relowner ''' try: - curs = db.query(qry) - - rows = [] - for row in curs.dictresult(): - rows.append(row) - - if len(rows) == 0: - logger.info('[OK] table ownership') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] table ownership') - logger.error('found %d table ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + rows = [] + for row in curs.fetchall(): + rows.append(row) + + if len(rows) == 0: + logger.info('[OK] table ownership') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] table ownership') + logger.error('found %d table ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s relowner %s != %s' % (row['nspname'], row['relname'], row['rolname'], @@ -1041,7 +1037,6 @@ def checkOwners(): # - Ignore implementation types of pg_class entries - they should be # in the check above since ALTER TABLE is required to fix them, not # ALTER TYPE. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, t.typname, a.rolname, m.rolname as coordinator_rolname from gp_dist_random('pg_type') r @@ -1052,27 +1047,28 @@ def checkOwners(): where r.typowner <> t.typowner ''' try: - curs = db.query(qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) - rows = [] - for row in curs.dictresult(): - rows.append(row) + rows = [] + for row in curs.fetchall(): + rows.append(row) - if len(rows) == 0: - logger.info('[OK] type ownership') - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - logger.info('[FAIL] type ownership') - logger.error('found %d type ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + if len(rows) == 0: + logger.info('[OK] type ownership') + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + logger.info('[FAIL] type ownership') + logger.error('found %d type ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s typeowner %s != %s' % (row['nspname'], row['typname'], row['rolname'], row['coordinator_rolname'])) if len(rows) > 100: logger.error("...") - except Exception as e: setError(ERROR_NOREPAIR) myprint("[ERROR] executing test: check type ownership") @@ -1096,15 +1092,14 @@ def checkOwners(): def closeDbs(): - for key, conns in GV.db.items(): - db = conns[0] - db.close() - GV.db = {} # remove everything + for key, conns in GV.conn.items(): + conn = conns[0] + conn.close() + GV.conn = {} # remove everything # ------------------------------------------------------------------------------- def getCatObj(namestr): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: cat = GV.catalog.getCatalogTable(namestr) except Exception as e: @@ -1174,25 +1169,25 @@ def checkTableACL(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - logger.info('[OK] Cross consistency acl check for ' + catname) - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - GV.aclStatus = False - logger.info('[FAIL] Cross consistency acl check for ' + catname) - logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) - - fields = curs.listfields() - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - processACLResult(catname, fields, curs.getresult()) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs = db.execute(qry) + nrows = curs.rowcount + if nrows == 0: + logger.info('[OK] Cross consistency acl check for ' + catname) + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + GV.aclStatus = False + logger.info('[FAIL] Cross consistency acl check for ' + catname) + logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) + + fields = [desc[0] for desc in curs.description] + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) + for row in curs.getresult(): + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) + processACLResult(catname, fields, curs.getresult()) except Exception as e: setError(ERROR_NOREPAIR) GV.aclStatus = False @@ -1213,9 +1208,9 @@ def checkForeignKey(cat_tables=None): if not cat_tables: cat_tables = GV.catalog.getCatalogTables() - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: - foreign_key_check = ForeignKeyCheck(db_connection, logger, GV.opt['-S'], autoCast) + foreign_key_check = ForeignKeyCheck(conn, logger, GV.opt['-S'], autoCast) foreign_key_issues = foreign_key_check.runCheck(cat_tables) if foreign_key_issues: GV.checkStatus = False @@ -1225,13 +1220,14 @@ def checkForeignKey(cat_tables=None): processForeignKeyResult(catname, pkcatname, fields, results) if catname == 'gp_fastsequence' and pkcatname == 'pg_class': setError(ERROR_REMOVE) - removeFastSequence(db_connection) + removeFastSequence(conn) else: setError(ERROR_NOREPAIR) except Exception as ex: setError(ERROR_NOREPAIR) GV.foreignKeyStatus = False myprint(' Execution error: ' + str(ex)) + # ------------------------------------------------------------------------------- @@ -1309,40 +1305,39 @@ def checkTableMissingEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - results = curs.getresult() - fields = curs.listfields() - - if nrows != 0: - results = filterSpuriousFailures(catname, fields, results) - nrows = len(results) - - if nrows == 0: - logger.info('[OK] Checking for missing or extraneous entries for ' + catname) - else: - if catname in ['pg_constraint']: - logger_with_level = logger.warning - log_level = logging.WARNING + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs.execute(qry) + nrows = curs.rowcount + results = curs.fetchall() + fields = [desc[0] for desc in curs.description] + + if nrows != 0: + results = filterSpuriousFailures(catname, fields, results) + nrows = len(results) + + if nrows == 0: + logger.info('[OK] Checking for missing or extraneous entries for ' + catname) else: - GV.checkStatus = False - GV.missingEntryStatus = False - logger_with_level = logger.error - log_level = logging.ERROR - - logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % - ('WARNING' if log_level == logging.WARNING else 'FAIL')) - logger_with_level(' %s has %d issue(s)' % (catname, nrows)) - gplog.log_literal(logger, log_level, " " + " | ".join(fields)) + if catname in ['pg_constraint']: + logger_with_level = logger.warning + log_level = logging.WARNING + else: + GV.checkStatus = False + GV.missingEntryStatus = False + logger_with_level = logger.error + log_level = logging.ERROR + + logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % + ('WARNING' if log_level == logging.WARNING else 'FAIL')) + logger_with_level(' %s has %d issue(s)' % (catname, nrows)) + gplog.log_literal(logger, log_level, " " + " | ".join(fields)) for row in results: gplog.log_literal(logger, log_level, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "missing") if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'missing_extraneous') - return results - except Exception as e: setError(ERROR_NOREPAIR) GV.missingEntryStatus = False @@ -1352,8 +1347,8 @@ def checkTableMissingEntry(cat): class checkAOSegVpinfoThread(execThread): - def __init__(self, cfg, db): - execThread.__init__(self, cfg, db, None) + def __init__(self, cfg, conn): + execThread.__init__(self, cfg, conn, None) def run(self): aoseg_query = """ @@ -1364,7 +1359,8 @@ class checkAOSegVpinfoThread(execThread): try: # Read the list of aoseg tables from the database - curs = self.db.query(aoseg_query) + curs = self.conn.cursor() + curs.execute(aoseg_query) for relname, relid, segrelid, segrelname in curs.getresult(): qry = "SELECT count(*) FROM pg_attribute WHERE attrelid=%d AND attnum > 0;" % (relid) @@ -1387,7 +1383,7 @@ class checkAOSegVpinfoThread(execThread): logger.error(qry) continue - vpinfo_length = vpinfo_curs.getresult()[0][0] + vpinfo_length = curs.fetchone()[0] # vpinfo is bytea type, the length of the first 3 fields is 12 bytes, and the size of AOCSVPInfoEntry is 16 # typedef struct AOCSVPInfo @@ -1420,8 +1416,8 @@ def checkAOSegVpinfo(): # parallelise check for dbid in GV.cfg: cfg = GV.cfg[dbid] - db_connection = connect2(cfg) - thread = checkAOSegVpinfoThread(cfg, db_connection) + conn = connect2(cfg) + thread = checkAOSegVpinfoThread(cfg, conn) thread.start() logger.debug('launching check thread %s for dbid %i' % (thread.getName(), dbid)) @@ -1435,6 +1431,83 @@ def checkAOSegVpinfo(): processThread(threads) +class checkAOLastrownumThread(execThread): + def __init__(self, cfg, conn): + execThread.__init__(self, cfg, conn, None) + + # pg_attribute_encoding.lastrownums[segno], if exists, should have a corresponding entry in gp_fastsequence with + # an objid same as segno. And the value of pg_attribute_encoding.lastrownums[segno] should fall in the range of + # [0, {last_sequence}] where {last_sequence} is the current gp_fastsequence value with the corresponding objid. + # Note that objmod starts from 0 but the array index starts from 1. + def run(self): + aolastrownum_query = """ + SELECT + c.relname, + ao.relid, + ae.attnum, + ae.lastrownums, + f.objmod, + f.last_sequence, + ae.lastrownums[f.objmod + 1] AS lastrownum + FROM + pg_attribute_encoding ae + JOIN pg_appendonly ao ON ae.attrelid = ao.relid + LEFT JOIN gp_fastsequence f ON ao.segrelid = f.objid + JOIN pg_class c ON ao.relid = c.oid + WHERE + f.last_sequence IS NULL + OR f.last_sequence < ae.lastrownums[f.objmod + 1] + OR ae.lastrownums[f.objmod + 1] < 0; + """ + + try: + # Execute the query + curs = self.conn.cursor() + curs.execute(aolastrownum_query) + nrows = curs.rowcount + + if nrows == 0: + logger.info('[OK] AO lastrownums check for pg_attribute_encoding') + else: + GV.checkStatus = False + # we could not fix this issue automatically + setError(ERROR_NOREPAIR) + logger.info('[FAIL] AO lastrownums check for pg_attribute_encoding') + for relname, relid, attnum, lastrownums, objmod, last_sequence, last_rownum in curs.fetchall(): + logger.error(" found inconsistent last_rownum {rownum} with last_sequence {seqnum} of aoseg {segno} for table '{relname}' attribute {attnum} on segment {content}" + .format(rownum = last_rownum, + seqnum = last_sequence, + segno = objmod, + relname = relname, + attnum = attnum, + content = self.cfg['content'])) + + except Exception as e: + GV.checkStatus = False + self.error = e + +# for test "ao_lastrownums" +def checkAOLastrownums(): + threads = [] + i = 1 + # parallelise check + for dbid in GV.cfg: + cfg = GV.cfg[dbid] + conn = connect2(cfg) + thread = checkAOLastrownumThread(cfg, conn) + thread.start() + logger.debug('launching check thread %s for dbid %i' % + (thread.name, dbid)) + threads.append(thread) + + if (i % GV.opt['-B']) == 0: + processThread(threads) + threads = [] + + i += 1 + + processThread(threads) + # ------------------------------------------------------------------------------- # Exclude these tuples from the catalog table scan @@ -1598,9 +1671,10 @@ def checkTableInconsistentEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for inconsistent entries for ' + catname) @@ -1611,16 +1685,14 @@ def checkTableInconsistentEntry(cat): logger.info('[FAIL] Checking for inconsistent entries for ' + catname) logger.error(' %s has %d issue(s)' % (catname, nrows)) - fields = curs.listfields() + fields = [desc[0] for desc in curs.description] gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): + results = curs.fetchall() + for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() processInconsistentEntryResult(catname, pkey, fields, results) if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'duplicate') - - except Exception as e: setError(ERROR_NOREPAIR) GV.inconsistentEntryStatus = False @@ -1734,9 +1806,10 @@ def checkTableDuplicateEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for duplicate entries for ' + catname) @@ -1748,7 +1821,7 @@ def checkTableDuplicateEntry(cat): fields = curs.listfields() gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - results = curs.getresult() + results = curs.fetchall() for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "duplicate") @@ -1797,9 +1870,9 @@ def duplicateEntryQuery(catname, pkey): def checkUniqueIndexViolation(): logger.info('-----------------------------------') logger.info('Performing check: checking for violated unique indexes') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - violations = UniqueIndexViolationCheck().runCheck(db_connection) + violations = UniqueIndexViolationCheck().runCheck(conn) checkname = 'unique index violation(s)' if violations: @@ -1836,9 +1909,9 @@ def checkOrphanedToastTables(): logger.info('-----------------------------------') logger.info('Performing check: checking for orphaned TOAST tables') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) checker = OrphanedToastTablesCheck() - check_passed = checker.runCheck(db_connection) + check_passed = checker.runCheck(conn) checkname = 'orphaned toast table(s)' if check_passed: @@ -2358,12 +2431,14 @@ def getOidFromPK(catname, pkeys): pkeystr=pkeystr) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - if (len(curs.dictresult()) == 0): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute(qry) + results = curs.fetchall() + if (len(results) == 0): raise QueryException("No such entry '%s' in %s" % (pkeystr, catname)) - return curs.dictresult().pop()['oid'] + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) @@ -2375,10 +2450,11 @@ def getOidFromPK(catname, pkeys): def getClassOidForRelfilenode(relfilenode): qry = "SELECT oid FROM pg_class WHERE relfilenode = %d;" % (relfilenode) try: - dburl = dbconn.DbURL(hostname=GV.opt['-h'], port=GV.opt['-p'], dbname=GV.dbname) - conn = dbconn.connect(dburl) - oid = dbconn.queryRow(conn, qry)[0] - return oid + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(qry) + oid = curs.fetchone()[0] + return oid except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -2398,10 +2474,12 @@ def getResourceTypeOid(oid): """ % (oid, oid) try: - db = connect() - curs = db.query(qry) - if len(curs.dictresult()) == 0: return 0 - return curs.dictresult().pop()['oid'] + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + results = curs.fetchall() + if len(results) == 0: return 0 + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -2996,7 +3074,7 @@ class GPObject: # Collect all tables with missing issues for later reporting if len(self.missingIssues): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) oid_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where oid=%d" type_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where reltype=%d" for issues in self.missingIssues.values() : @@ -3004,19 +3082,23 @@ class GPObject: # Get schemaname.tablename corresponding to oid for key in issue.pkeys: if 'relid' in key or key in ['ev_class', 'reloid']: - table_list = db.query(oid_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(oid_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) elif key == 'oid': - table_list = db.query(type_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(type_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) def __cmp__(self, other): @@ -3165,9 +3247,11 @@ def getRelInfo(objects): """.format(oids=','.join(map(str, oids))) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - for row in curs.getresult(): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + results = curs.fetchall() + for row in results: (oid, relname, nspname, relkind, paroid) = row objects[oid, 'pg_class'].setRelInfo(relname, nspname, relkind, paroid) diff --git a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py index da9272c6c12..04a7a69ba56 100644 --- a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py @@ -2,6 +2,7 @@ from gppylib.gplog import * from gppylib.gpcatalog import * +from contextlib import closing import re class ForeignKeyCheck: @@ -116,25 +117,25 @@ def checkTableForeignKey(self, cat): def _validate_relation(self, catname, fkeystr, pkcatname, pkeystr, qry): issue_list = [] try: - curs = self.db_connection.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - else: - self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % - (catname, nrows, pkcatname, pkeystr)) - - fields = curs.listfields() - log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() - issue_list.append((pkcatname, fields, results)) - + with closing(self.db_connection.cursor()) as curs: + curs.execute(qry) + nrows = curs.rowcount + + if nrows == 0: + self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + else: + self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % + (catname, nrows, pkcatname, pkeystr)) + + fields = [desc[0] for desc in curs.description] + log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) + results = curs.fetchall() + for row in results: + log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) + issue_list.append((pkcatname, fields, results)) except Exception as e: err_msg = '[ERROR] executing: Foreign key check for catalog table {0}. Query : \n {1}\n'.format(catname, qry) err_msg += str(e) diff --git a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py index 87e55a5cf7b..dc7cfacb32f 100644 --- a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py +++ b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py @@ -35,16 +35,19 @@ class LeakedSchemaDropper: """ def __get_leaked_schemas(self, db_connection): - leaked_schemas = db_connection.query(self.leaked_schema_query) + with db_connection.cursor() as curs: + curs.execute(self.leaked_schema_query) + leaked_schemas = curs.fetchall() - if not leaked_schemas: - return [] + if not leaked_schemas: + return [] - return [row[0] for row in leaked_schemas.getresult() if row[0]] + return [row[0] for row in leaked_schemas if row[0]] def drop_leaked_schemas(self, db_connection): leaked_schemas = self.__get_leaked_schemas(db_connection) for leaked_schema in leaked_schemas: escaped_schema_name = escapeDoubleQuoteInSQLString(leaked_schema) - db_connection.query('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) + with db_connection.cursor() as curs: + curs.execute('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) return leaked_schemas diff --git a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py index 21ec8d18047..a76ef560867 100644 --- a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py @@ -4,6 +4,8 @@ from collections import namedtuple from gpcheckcat_modules.orphan_toast_table_issues import OrphanToastTableIssue, DoubleOrphanToastTableIssue, ReferenceOrphanToastTableIssue, DependencyOrphanToastTableIssue, MismatchOrphanToastTableIssue +import psycopg2 +from psycopg2 import extras OrphanedTable = namedtuple('OrphanedTable', 'oid catname') @@ -117,8 +119,10 @@ def __init__(self): """ def runCheck(self, db_connection): - orphaned_toast_tables = db_connection.query(self.orphaned_toast_tables_query).dictresult() - if len(orphaned_toast_tables) == 0: + curs = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute(self.orphaned_toast_tables_query) + orphaned_toast_tables = curs.fetchall() + if curs.rowcount == 0: return True for row in orphaned_toast_tables: diff --git a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py index 47999f5c59c..6778401f31a 100644 --- a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py @@ -34,22 +34,24 @@ def __init__(self): ) as violations """ - def runCheck(self, db_connection): - unique_indexes = db_connection.query(self.unique_indexes_query).getresult() - violations = [] + def runCheck(self, conn): + with conn.cursor() as cur: + cur.execute(self.unique_indexes_query) + unique_indexes = cur.fetchall() + violations = [] - for (table_oid, index_name, table_name, column_names) in unique_indexes: - column_names = ",".join(column_names) - sql = self.get_violated_segments_query(table_name, column_names) - violated_segments = db_connection.query(sql).getresult() - if violated_segments: - violations.append(dict(table_oid=table_oid, - table_name=table_name, - index_name=index_name, - column_names=column_names, - violated_segments=[row[0] for row in violated_segments])) - - return violations + for (table_oid, index_name, table_name, column_names) in unique_indexes: + column_names = ",".join(column_names) + sql = self.get_violated_segments_query(table_name, column_names) + cur.execute(sql) + violated_segments = cur.fetchall() + if violated_segments: + violations.append(dict(table_oid=table_oid, + table_name=table_name, + index_name=index_name, + column_names=column_names, + violated_segments=[row[0] for row in violated_segments])) + return violations def get_violated_segments_query(self, table_name, column_names): return self.violated_segments_query % ( diff --git a/gpMgmt/bin/gpconfig b/gpMgmt/bin/gpconfig index 7bd3023ea85..464e9d79440 100755 --- a/gpMgmt/bin/gpconfig +++ b/gpMgmt/bin/gpconfig @@ -15,7 +15,7 @@ import os import sys import re - +from psycopg2 import DatabaseError try: from gppylib.gpparseopts import OptParser, OptChecker from gppylib.gparray import GpArray @@ -25,7 +25,6 @@ try: from gppylib.commands.gp import * from gppylib.db import dbconn from gppylib.userinput import * - from pg import DatabaseError from gpconfig_modules.segment_guc import SegmentGuc from gpconfig_modules.database_segment_guc import DatabaseSegmentGuc from gpconfig_modules.file_segment_guc import FileSegmentGuc diff --git a/gpMgmt/bin/gpexpand b/gpMgmt/bin/gpexpand index 3825095829a..368d63ffb96 100755 --- a/gpMgmt/bin/gpexpand +++ b/gpMgmt/bin/gpexpand @@ -17,10 +17,10 @@ import signal import traceback from collections import defaultdict from time import strftime, sleep - +import psycopg2 +from psycopg2 import DatabaseError, OperationalError +from psycopg2 import extras try: - import pg, pgdb - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.gparray import GpArray, MODE_NOT_SYNC, STATUS_DOWN @@ -32,9 +32,9 @@ try: from gppylib.operations.startSegments import MIRROR_MODE_MIRRORLESS from gppylib.system import configurationInterface, configurationImplGpdb from gppylib.system.environment import GpCoordinatorEnvironment - from pgdb import DatabaseError from gppylib.gpcatalog import COORDINATOR_ONLY_TABLES from gppylib.operations.package import SyncPackages + from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup from gppylib.operations.utils import ParallelOperation from gppylib.parseutils import line_reader, check_values, canonicalize_address from gppylib.heapchecksum import HeapChecksum @@ -49,6 +49,9 @@ except ImportError as e: MAX_PARALLEL_EXPANDS = 96 MAX_BATCH_SIZE = 128 +FIRST_NORMAL_OID = 16384 +RELKIND_FOREIGN_TABLE = 'f' + SEGMENT_CONFIGURATION_BACKUP_FILE = "gpexpand.gp_segment_configuration" DBNAME = 'postgres' @@ -259,6 +262,17 @@ def gpexpand_status_file_exists(coordinator_data_directory): return os.path.exists(coordinator_data_directory + '/gpexpand.status') +def is_cluster_up_and_balanced(dburl): + count = -1 + sql = "select count(*) from gp_segment_configuration where status <> 'u' or preferred_role <> role;" + try: + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + count = dbconn.querySingleton(conn, sql) + except Exception as e: + raise Exception("failed to query cluster role check: %s" % str(e)) + + return count == 0 + # ------------------------------------------------------------------------- # expansion schema @@ -275,16 +289,16 @@ status_table_sql = """CREATE TABLE gpexpand.status updated timestamp ) """ status_detail_table_sql = """CREATE TABLE gpexpand.status_detail - ( dbname text, + ( table_oid oid, + dbname text, fq_name text, - table_oid oid, root_partition_oid oid, rank int, external_writable bool, status text, expansion_started timestamp, expansion_finished timestamp, - source_bytes numeric ) """ + source_bytes numeric ) distributed by (table_oid)""" # gpexpand views progress_view_simple_sql = """CREATE VIEW gpexpand.expansion_progress AS SELECT @@ -1143,6 +1157,13 @@ class gpexpand: return dict(cursor.fetchall()) def addNewSegments(self, inputFileEntryList): + segments_with_running_basebackup = get_segments_with_running_basebackup() + contentIds_with_running_basebackup = [seg.contentId for seg in inputFileEntryList if seg.contentId in segments_with_running_basebackup] + + if len(contentIds_with_running_basebackup)>0: + raise Exception( + "Found pg_basebackup running for segments with contentIds %s" %contentIds_with_running_basebackup) + for seg in inputFileEntryList: self.gparray.addExpansionSeg(content=int(seg.contentId) , preferred_role=seg.role @@ -1596,32 +1617,82 @@ class gpexpand: # including new segments has been started once before so finalize self.finalize_prepare() + def generate_sql_to_populate_table_size(self, sql): + # This function generate a SQL that can compute table size + # taking the advantage of MPP. Previous code put pg_relation_size + # in target list of a query involving only catalog, thus each + # tuple will lead to a dispatch to compute pg_relation_size + # thus bad performance. Now we let pg_relation_size compute + # at each segment and then group by the table oid and sum + # together to get each table's size. pg_relation_size is + # volatile function, so the following CTE sql's subquery + # x will never be pulled up which means the pg_relation_size + # will always be evaluated before later motion to gurantee + # correctness no matter single stage or multi stage agg. + # + # Also note, we filter out those relations with relkind is + # RELKIND_FOREIGN_TABLE, because we only expand those + # are external writable, and even for external writable ones + # we simple modify the numsegments and do not move data. + # Refer to C code: ATExecExpandTable and ATExecExpandPartitionTablePrepare. + cte_sql = """with table_size_cte(table_oid, size) as + ( + select table_oid, sum(size) + from ( + select oid as table_oid, + pg_relation_size(oid) as size + from gp_dist_random('pg_class') + where oid >= %d and relkind <> '%s' + ) x(table_oid, size) + group by table_oid + ) + """ % (FIRST_NORMAL_OID, RELKIND_FOREIGN_TABLE) + + final_sql = """{cte_sql} + select + s1.table_oid, + s1.dbname, + s1.fq_name, + s1.root_partition_oid, + s1.rank, + s1.external_writable, + s1.undone_status, + s1.expansion_started, + s1.expansion_finished, + coalesce(table_size_cte.size, 0) as source_bytes + from ({orig_sql})s1 left join table_size_cte + on s1.table_oid = table_size_cte.table_oid + """ + return final_sql.format(cte_sql=cte_sql, + orig_sql=sql) + def _populate_regular_tables(self, dbname): - src_bytes_str = "0" if self.options.simple_progress else "pg_relation_size(quote_ident(n.nspname) || '.' || quote_ident(c.relname))" sql = """SELECT - current_database(), - quote_ident(n.nspname) || '.' || quote_ident(c.relname) as fq_name, - c.oid as tableoid, - NULL as root_partition_oid, - 2 as rank, - pe.writable is not null as external_writable, - '%s' as undone_status, - NULL as expansion_started, - NULL as expansion_finished, - %s as source_bytes -FROM - pg_class c - JOIN pg_namespace n ON (c.relnamespace=n.oid) - JOIN pg_catalog.gp_distribution_policy p on (c.oid = p.localoid) - LEFT JOIN pg_partitioned_table pp on (c.oid=pp.partrelid) - LEFT JOIN pg_exttable pe on (c.oid=pe.reloid and pe.writable) -WHERE - pp.partrelid is NULL - AND NOT c.relispartition - AND n.nspname != 'gpexpand' - AND n.nspname != 'pg_bitmapindex' - AND c.relpersistence != 't' - """ % (undone_status, src_bytes_str) + c.oid as table_oid, + current_database() as dbname, + quote_ident(n.nspname) || '.' || quote_ident(c.relname) as fq_name, + NULL as root_partition_oid, + 2 as rank, + pe.writable is not null as external_writable, + '%s' as undone_status, + NULL as expansion_started, + NULL as expansion_finished, + 0 as source_bytes + FROM + pg_class c + JOIN pg_namespace n ON (c.relnamespace=n.oid) + JOIN pg_catalog.gp_distribution_policy p on (c.oid = p.localoid) + LEFT JOIN pg_partitioned_table pp on (c.oid=pp.partrelid) + LEFT JOIN pg_exttable pe on (c.oid=pe.reloid and pe.writable) + WHERE + pp.partrelid is NULL + AND NOT c.relispartition + AND n.nspname != 'gpexpand' + AND n.nspname != 'pg_bitmapindex' + AND c.relpersistence != 't' """ % undone_status + + if not self.options.simple_progress: + sql = self.generate_sql_to_populate_table_size(sql) self.logger.debug(sql) table_conn = self.connect_database(dbname) @@ -1684,19 +1755,18 @@ WHERE self.logger.debug(prepare_cmd) dbconn.execSQL(table_conn, prepare_cmd, autocommit=False) - src_bytes_str = "0" if self.options.simple_progress else "pg_relation_size(c.oid)" get_status_detail_cmd = """ SELECT - current_database(), + c.oid as table_oid, + current_database() as dbname, quote_ident(n.nspname) || '.' || quote_ident(c.relname) as fq_name, - c.oid as tableoid, d.oid as root_partition_oid, 2 as rank, false as external_writable, '%s' as undone_status, NULL as expansion_started, NULL as expansion_finished, - %s as source_bytes + 0 as source_bytes FROM pg_inherits a, pg_partitioned_table b, @@ -1710,7 +1780,10 @@ WHERE c.relnamespace = n.oid and c.relkind != 'p' and c.relkind != 'f' - """ % (undone_status, src_bytes_str) + """ % undone_status + + if not self.options.simple_progress: + get_status_detail_cmd = self.generate_sql_to_populate_table_size(get_status_detail_cmd) self.logger.debug(get_status_detail_cmd) try: @@ -1836,7 +1909,7 @@ WHERE expansionStopped) dbconn.execSQL(self.conn, sql) self.conn.close() - except pgdb.OperationalError: + except OperationalError: pass except Exception: # schema doesn't exist. Cancel or error during setup @@ -1888,7 +1961,7 @@ WHERE def connect_database(self, dbname): test_url = copy.deepcopy(self.dburl) test_url.pgdb = dbname - c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True) + c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True, cursorFactory=psycopg2.extras.NamedTupleCursor) return c def sync_packages(self): @@ -1947,7 +2020,7 @@ class ExpandTable(): def __init__(self, options, row=None): self.options = options if row is not None: - (self.dbname, self.fq_name, self.table_oid, + (self.table_oid, self.dbname, self.fq_name, self.root_partition_oid, self.rank, self.external_writable, self.status, self.expansion_started, self.expansion_finished, @@ -2129,7 +2202,7 @@ class ExpandCommand(SQLCommand): try: status_conn = dbconn.connect(self.status_url, encoding='UTF8') - table_conn = dbconn.connect(self.table_url, encoding='UTF8') + table_conn = dbconn.connect(self.table_url, encoding='UTF8', cursorFactory=psycopg2.extras.NamedTupleCursor) except DatabaseError as ex: if self.options.verbose: logger.exception(ex) @@ -2460,6 +2533,10 @@ def main(options, args, parser): logger.error(e) sys.exit(1) + # check if the cluster is in good health + if gpexpand_file_status is None and gpexpand_db_status is None and not is_cluster_up_and_balanced(dburl): + logger.warning('One or more segments are either down or not in preferred role.') + if gpexpand_db_status == 'SETUP DONE' or gpexpand_db_status == 'EXPANSION STOPPED': if not _gp_expand.validate_max_connections(): raise ValidationError() diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index 2d7c0a1eb7f..b887a8eeb6b 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -35,22 +35,8 @@ sys.exit(2) import platform - -try: - import pg -except ImportError: - try: - from pygresql import pg - except Exception as e: - pass -except Exception as e: - print(repr(e)) - errorMsg = "gpload was unable to import The PyGreSQL Python module (pg.py) - %s\n" % str(e) - sys.stderr.write(str(errorMsg)) - errorMsg = "Please check if you have the correct Visual Studio redistributable package installed.\n" - sys.stderr.write(str(errorMsg)) - sys.exit(2) - +import psycopg2 +from psycopg2 import extras import hashlib import datetime,getpass,os,signal,socket,threading,time,traceback,re import subprocess @@ -562,6 +548,8 @@ def is_keyword(tab): else: return False +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def caseInsensitiveDictLookup(key, dictionary): """ diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 7daae009cc1..f78e295e144 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -10,10 +10,8 @@ import os import sys import signal import itertools - +from psycopg2 import DatabaseError try: - import pg - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.commands.pg import PgControlData @@ -23,7 +21,6 @@ try: from gppylib.db import dbconn from gppylib.userinput import * from gppylib.operations.startSegments import * - from pgdb import DatabaseError from gppylib import gparray, gplog, pgconf, userinput, utils from gppylib.parseutils import line_reader, check_values, canonicalize_address from gppylib.operations.segment_tablespace_locations import get_tablespace_locations @@ -373,12 +370,12 @@ try: """ Prepare common execution steps for running commands on segments """ - mirrorsToDelete = [mirror for mirror in newConfig.oldMirrorList if not mirror.inPlace] - totalDirsToDelete = len(mirrorsToDelete) + oldMirrorsToMove = [mirror for mirror in newConfig.oldMirrorList if not mirror.inPlace] + totalDirsToDelete = len(oldMirrorsToMove) numberOfWorkers = min(totalDirsToDelete, options.batch_size) """ Fetch tablespace locations before mirror move """ - for mirror in mirrorsToDelete: + for mirror in oldMirrorsToMove: mirror_tablespace_locations[mirror.dataDirectory] = get_tablespace_locations(False, mirror.dataDirectory) """ Update pg_hba.conf on primary segments with new mirror information """ @@ -400,6 +397,10 @@ try: """ Reinitialize gparray after the new mirrors are in place. """ gpArrayInstance = GpArray.initFromCatalog(dburl, utility=True) mirrorDirectories = set((seg.getSegmentHostName(),seg.getSegmentContentId()) for seg in gpArrayInstance.getDbList() if seg.isSegmentMirror()) + newMirrorAddressDir = set((seg.getSegmentAddress(), seg.getSegmentDataDirectory()) for seg in gpArrayInstance.getDbList() if seg.isSegmentMirror()) + + """If mirror has not moved to new location due to any reason, skip deletion of old mirror directory""" + mirrorsToDelete = [mirror for mirror in oldMirrorsToMove if (mirror.address, mirror.dataDirectory) not in newMirrorAddressDir] """ Delete old mirror directories. """ if numberOfWorkers > 0: diff --git a/gpMgmt/bin/gppylib/commands/base.py b/gpMgmt/bin/gppylib/commands/base.py index 0e0296d7c4c..aae6194c1a1 100755 --- a/gpMgmt/bin/gppylib/commands/base.py +++ b/gpMgmt/bin/gppylib/commands/base.py @@ -29,7 +29,6 @@ from gppylib import gplog from gppylib import gpsubprocess -from pg import DB logger = gplog.get_default_logger() @@ -629,8 +628,7 @@ def cancel(self): # if self.conn is not set we cannot cancel. if self.cancel_conn: - DB(self.cancel_conn).cancel() - + self.cancel_conn.cancel() def run_remote_commands(name, commands): """ diff --git a/gpMgmt/bin/gppylib/commands/pg.py b/gpMgmt/bin/gppylib/commands/pg.py index 23aba4660de..0bb00b2a6fd 100644 --- a/gpMgmt/bin/gppylib/commands/pg.py +++ b/gpMgmt/bin/gppylib/commands/pg.py @@ -12,6 +12,7 @@ from .unix import * from gppylib.commands.base import * from gppylib.commands.gp import RECOVERY_REWIND_APPNAME +from psycopg2 import DatabaseError logger = get_default_logger() diff --git a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py index 117f62b41ea..59b0f05beba 100644 --- a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py +++ b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py @@ -1,5 +1,7 @@ import unittest from gppylib.commands import pg +from test.unit.gp_unittest import GpTestCase, run_tests +from psycopg2 import DatabaseError class TestUnitPgBaseBackup(unittest.TestCase): diff --git a/gpMgmt/bin/gppylib/db/catalog.py b/gpMgmt/bin/gppylib/db/catalog.py index 6214b805c29..80ca66289eb 100644 --- a/gpMgmt/bin/gppylib/db/catalog.py +++ b/gpMgmt/bin/gppylib/db/catalog.py @@ -6,8 +6,8 @@ """ import copy - -import pg +import os +from contextlib import closing from gppylib import gplog from gppylib.db import dbconn diff --git a/gpMgmt/bin/gppylib/db/dbconn.py b/gpMgmt/bin/gppylib/db/dbconn.py index 7d3f839f68b..29959e356c6 100644 --- a/gpMgmt/bin/gppylib/db/dbconn.py +++ b/gpMgmt/bin/gppylib/db/dbconn.py @@ -9,9 +9,8 @@ import sys import os import stat - +import psycopg2 try: - import pgdb from gppylib.commands.unix import UserId except ImportError as e: @@ -159,51 +158,36 @@ def canonicalize(s): # 1. pg notice is accessible to a user of connection returned by dbconn.connect(), # lifted from the underlying _pg connection # 2. multiple calls to dbconn.close() should not return an error -class Connection(pgdb.Connection): +class Connection: def __init__(self, connection): - self._notices = collections.deque(maxlen=100) - # we must do an attribute by attribute copy of the notices here - # due to limitations in pg implementation. Wrap with with a - # namedtuple for ease of use. - def handle_notice(notice): - received = {} - for attr in dir(notice): - if attr.startswith('__'): - continue - value = getattr(notice, attr) - received[attr] = value - Notice = collections.namedtuple('Notice', sorted(received)) - self._notices.append(Notice(**received)) - - - self._impl = connection - self._impl._cnx.set_notice_receiver(handle_notice) + self._conn = connection + self._conn.notices = collections.deque(maxlen=100) def __enter__(self): - return self._impl.__enter__() + return self._conn.__enter__() # __exit__() does not close the connection. This is in line with the # python DB API v2 specification (pep-0249), where close() is done on # __del__(), not __exit__(). def __exit__(self, *args): - return self._impl.__exit__(*args) + return self._conn.__exit__(*args) def __getattr__(self, name): - return getattr(self._impl, name) + return getattr(self._conn, name) def notices(self): - notice_list = list(self._notices) - self._notices.clear() + notice_list = list(self._conn.notices) + self._conn.notices.clear() return notice_list # don't return operational error if connection is already closed def close(self): - if not self._impl.closed: - self._impl.close() + if not self._conn.closed: + self._conn.close() def connect(dburl, utility=False, verbose=False, - encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True): + encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True, cursorFactory=None): conninfo = { 'user': dburl.pguser, @@ -211,6 +195,7 @@ def connect(dburl, utility=False, verbose=False, 'host': dburl.pghost, 'port': dburl.pgport, 'database': dburl.pgdb, + 'cursor_factory': cursorFactory } # building options @@ -248,22 +233,23 @@ def connect(dburl, utility=False, verbose=False, logFunc = logger.info if dburl.timeout is not None else logger.debug logFunc("Connecting to db {} on host {}".format(dburl.pgdb, dburl.pghost)) - connection = None + conn = None for i in range(retries): try: - connection = pgdb.connect(**conninfo) + conn = psycopg2.connect(**conninfo) + conn.set_session(autocommit=True) break - except pgdb.OperationalError as e: + except psycopg2.OperationalError as e: if 'timeout expired' in str(e): logger.warning('Timeout expired connecting to %s, attempt %d/%d' % (dburl.pgdb, i+1, retries)) continue raise - if connection is None: + if conn is None: raise ConnectionError('Failed to connect to %s' % dburl.pgdb) - return Connection(connection) + return Connection(conn) def execSQL(conn, sql, autocommit=True): """ @@ -277,7 +263,6 @@ def execSQL(conn, sql, autocommit=True): Using with `dbconn.connect() as conn` syntax will override autocommit and complete queries in a transaction followed by a commit on context close """ - conn.autocommit = autocommit with conn.cursor() as cursor: cursor.execute(sql) diff --git a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py index 086ccce9298..a03eb4db42b 100644 --- a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py +++ b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py @@ -46,7 +46,7 @@ def test_verbose_mode_allows_warnings_to_be_sent_to_the_client(self): for notice in notices: - if warning in notice.message: + if warning in notice: return # found it! self.fail("Didn't find expected notice '{}' in {!r}".format( diff --git a/gpMgmt/bin/gppylib/gparray.py b/gpMgmt/bin/gppylib/gparray.py index 83545fb82e8..5f39434b83b 100755 --- a/gpMgmt/bin/gppylib/gparray.py +++ b/gpMgmt/bin/gppylib/gparray.py @@ -1290,54 +1290,66 @@ def get_segment_count(self): return len(self.segmentPairs) # -------------------------------------------------------------------- - def get_min_primary_port(self): - """Returns the minimum primary segment db port""" - min_primary_port = self.segmentPairs[0].primaryDB.port + def get_primary_port_list(self): + primary_ports = [] for segPair in self.segmentPairs: - if segPair.primaryDB.port < min_primary_port: - min_primary_port = segPair.primaryDB.port + primary = segPair.primaryDB + mirror = segPair.mirrorDB + + if primary.preferred_role == primary.role: + primary_ports.append(primary.port) + else: + primary_ports.append(mirror.port) - return min_primary_port + if len(primary_ports) == 0: + raise Exception("No primary ports found in array.") + return primary_ports # -------------------------------------------------------------------- - def get_max_primary_port(self): - """Returns the maximum primary segment db port""" - max_primary_port = self.segmentPairs[0].primaryDB.port - for segPair in self.segmentPairs: - if segPair.primaryDB.port > max_primary_port: - max_primary_port = segPair.primaryDB.port - return max_primary_port + def get_mirror_port_list(self): - # -------------------------------------------------------------------- - def get_min_mirror_port(self): - """Returns the minimum mirror segment db port""" if self.get_mirroring_enabled() is False: raise Exception('Mirroring is not enabled') - min_mirror_port = self.segmentPairs[0].mirrorDB.port - + mirror_ports = [] for segPair in self.segmentPairs: + primary = segPair.primaryDB mirror = segPair.mirrorDB - if mirror and mirror.port < min_mirror_port: - min_mirror_port = mirror.port - return min_mirror_port + if mirror.preferred_role == mirror.role: + mirror_ports.append(mirror.port) + else: + mirror_ports.append(primary.port) + + if len(mirror_ports) == 0: + raise Exception("No mirror ports found in array.") + + return mirror_ports # -------------------------------------------------------------------- - def get_max_mirror_port(self): - """Returns the maximum mirror segment db port""" - if self.get_mirroring_enabled() is False: - raise Exception('Mirroring is not enabled') + def get_min_primary_port(self): + """Returns the minimum primary segment db port""" + primary_ports = self.get_primary_port_list() + return min(primary_ports) - max_mirror_port = self.segmentPairs[0].mirrorDB.port + # -------------------------------------------------------------------- + def get_max_primary_port(self): + """Returns the maximum primary segment db port""" + primary_ports = self.get_primary_port_list() + return max(primary_ports) - for segPair in self.segmentPairs: - mirror = segPair.mirrorDB - if mirror and mirror.port > max_mirror_port: - max_mirror_port = mirror.port + # -------------------------------------------------------------------- + def get_min_mirror_port(self): + """Returns the minimum mirror segment db port""" + mirror_ports = self.get_mirror_port_list() + return min(mirror_ports) - return max_mirror_port + # -------------------------------------------------------------------- + def get_max_mirror_port(self): + """Returns the maximum mirror segment db port""" + mirror_ports = self.get_mirror_port_list() + return max(mirror_ports) # -------------------------------------------------------------------- def get_interface_numbers(self): diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index 1a7325daff6..611244f4f5f 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -144,7 +144,7 @@ def __init__(self, dbConnection): curs = self._query(version_query) except Exception as e: raise GPCatalogException("Error reading database version: " + str(e)) - self._version = GpVersion(curs.getresult()[0][0]) + self._version = GpVersion(curs.fetchone()[0]) # Read the list of catalog tables from the database try: @@ -154,7 +154,7 @@ def __init__(self, dbConnection): # Construct our internal representation of the catalog - for [oid, relname, relisshared] in curs.getresult(): + for [oid, relname, relisshared] in curs.fetchall(): self._tables[relname] = GPCatalogTable(self, relname) # Note: stupid API returns t/f for boolean value self._tables[relname]._setShared(relisshared == 't') @@ -193,7 +193,9 @@ def _query(self, qry): """ Simple wrapper around querying the database connection """ - return self._dbConnection.query(qry) + cur = self._dbConnection.cursor() + cur.execute(qry) + return cur def _markCoordinatorOnlyTables(self): """ @@ -483,10 +485,10 @@ def __init__(self, parent, name, pkey=None): # exist. raise GPCatalogException("Catalog table %s does not exist" % name) - if cur.ntuples() == 0: + if cur.rowcount == 0: raise GPCatalogException("Catalog table %s does not exist" % name) - for row in cur.getresult(): + for row in cur.fetchall(): (attname, atttype, typname) = row # Mark if the catalog has an oid column @@ -522,7 +524,7 @@ def __init__(self, parent, name, pkey=None): WHERE attrelid = 'pg_catalog.{catname}'::regclass """.format(catname=name) cur = parent._query(qry) - self._pkey = [row[0] for row in cur.getresult()] + self._pkey = [row[0] for row in cur.fetchall()] # Primary key must be in the column list for k in self._pkey: diff --git a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py index 643489fd086..913847a5eb5 100644 --- a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py @@ -1,8 +1,7 @@ import time - from gppylib.commands import base from gppylib.db import dbconn -import pg +from contextlib import closing FTS_PROBE_QUERY = 'SELECT pg_catalog.gp_request_fts_probe_scan()' @@ -13,15 +12,19 @@ def __init__(self, logger, worker_pool, timeout): self.timeout = timeout def _trigger_fts_probe(self, dburl): - conn = pg.connect(dbname=dburl.pgdb, - host=dburl.pghost, - port=dburl.pgport, - opt=None, - user=dburl.pguser, - passwd=dburl.pgpass, - ) - conn.query(FTS_PROBE_QUERY) - conn.close() + start_time = time.time() + while True: + try: + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute(FTS_PROBE_QUERY) + break + except Exception as e: + now = time.time() + if now < start_time + self.timeout: + continue + else: + raise RuntimeError("FTS probing did not complete in {} seconds.".format(self.timeout)) def reconfigure(self): # issue a distributed query to make sure we pick up the fault @@ -36,9 +39,12 @@ def reconfigure(self): # Empty block of 'BEGIN' and 'END' won't start a distributed transaction, # execute a DDL query to start a distributed transaction. # so the primaries'd better be up - conn = dbconn.connect(dburl) - conn.cursor().execute('CREATE TEMP TABLE temp_test(a int)') - conn.cursor().execute('COMMIT') + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute('BEGIN') + cur.execute('CREATE TEMP TABLE temp_test(a int)') + cur.execute('COMMIT') + break except Exception as e: # Should close conn here # Otherwise, the postmaster will be blocked by abort transaction @@ -48,6 +54,3 @@ def reconfigure(self): continue else: raise RuntimeError("Mirror promotion did not complete in {0} seconds.".format(self.timeout)) - else: - conn.close() - break diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py index 05bfeb55e56..9004e40e8dc 100644 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py @@ -4,8 +4,7 @@ from gppylib.operations.segment_reconfigurer import SegmentReconfigurer, FTS_PROBE_QUERY from gppylib.test.unit.gp_unittest import GpTestCase -import pg -import pgdb +import psycopg2 import mock from mock import Mock, patch, call, MagicMock import contextlib @@ -38,22 +37,22 @@ def setUp(self): self.apply_patches([ patch('gppylib.db.dbconn.connect', new=self.connect), patch('gppylib.db.dbconn.DbURL', return_value=self.db_url), - patch('pg.connect'), + patch('psycopg2.connect'), ]) def test_it_triggers_fts_probe(self): reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) reconfigurer.reconfigure() - pg.connect.assert_has_calls([ - call(dbname=self.db, host=self.host, port=self.port, opt=None, user=self.user, passwd=self.passwd), + psycopg2.connect.assert_has_calls([ + call(dbname=self.db, host=self.host, port=self.port, options=None, user=self.user, password=self.passwd), call().query(FTS_PROBE_QUERY), call().close(), ] ) def test_it_retries_the_connection(self): - self.connect.configure_mock(side_effect=[pgdb.DatabaseError, pgdb.DatabaseError, self.conn]) + self.connect.configure_mock(side_effect=[psycopg2.DatabaseError, psycopg2.DatabaseError, self.conn]) reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) @@ -74,7 +73,7 @@ def fail_for_half_a_minute(): # leap forward 15 seconds new_time += self.timeout / 2 now_mock.configure_mock(return_value=new_time) - yield pgdb.DatabaseError + yield psycopg2.DatabaseError self.connect.configure_mock(side_effect=fail_for_half_a_minute()) @@ -87,3 +86,27 @@ def fail_for_half_a_minute(): self.connect.assert_has_calls([call(self.db_url), call(self.db_url), ]) self.conn.close.assert_has_calls([]) + + @patch('time.time') + def test_it_gives_up_after_600_seconds_2(self, now_mock): + start_datetime = datetime.datetime(2023, 7, 27, 16, 0, 0) + start_time = time.mktime(start_datetime.timetuple()) + now_mock.configure_mock(return_value=start_time) + + def fail_for_ten_minutes(): + new_time = start_time + # leap forward 600 seconds + new_time += self.timeout + now_mock.configure_mock(return_value=new_time) + yield psycopg2.DatabaseError + + self.connect.configure_mock(side_effect=fail_for_ten_minutes()) + + reconfigurer = SegmentReconfigurer(logger=self.logger, + worker_pool=self.worker_pool, timeout=self.timeout) + with self.assertRaises(RuntimeError) as context: + reconfigurer.reconfigure() + self.assertEqual("FTS probing did not complete in {} seconds.".format(self.timeout), context.exception.message) + + self.connect.assert_has_calls([call(self.db_url)]) + self.conn.close.assert_has_calls([]) diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py index 38fcd327dee..e62aa7a40b7 100755 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py @@ -11,7 +11,7 @@ from gppylib.operations.test_utils_helper import TestOperation, RaiseOperation, RaiseOperation_Unpicklable, RaiseOperation_Safe, ExceptionWithArgs from operations.unix import ListFiles from test.unit.gp_unittest import GpTestCase, run_tests -from pg import DatabaseError +from psycopg2 import DatabaseError class UtilsTestCase(GpTestCase): """ diff --git a/gpMgmt/bin/gppylib/operations/test_utils_helper.py b/gpMgmt/bin/gppylib/operations/test_utils_helper.py index 109bc83bb87..74cfeda2bc0 100755 --- a/gpMgmt/bin/gppylib/operations/test_utils_helper.py +++ b/gpMgmt/bin/gppylib/operations/test_utils_helper.py @@ -1,4 +1,5 @@ from gppylib.operations import Operation +import psycopg2 """ These objects needed for gppylib.operations.test.test_utils are pulled out of said file for @@ -37,5 +38,4 @@ def __init__(self, x, y): class RaiseOperation_Unpicklable(Operation): def execute(self): - import pg - raise pg.DatabaseError() + raise psycopg2.DatabaseError() diff --git a/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py b/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py index 34b6a12ff2e..5d925b35b9a 100644 --- a/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py +++ b/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py @@ -1,12 +1,37 @@ import abc from typing import List +from contextlib import closing +from gppylib.db import dbconn +from gppylib import gplog from gppylib.mainUtils import ExceptionNoStackTraceNeeded from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts from gppylib.parseutils import line_reader, check_values, canonicalize_address from gppylib.utils import checkNotNone, normalizeAndValidateInputPath from gppylib.gparray import GpArray, Segment +logger = gplog.get_default_logger() + +def get_segments_with_running_basebackup(): + """ + Returns a list of contentIds of source segments of running pg_basebackup processes + At present gp_stat_replication table does not contain any info about datadir and dbid of the target of running pg_basebackup + """ + + sql = "select gp_segment_id from gp_stat_replication where application_name = 'pg_basebackup'" + + try: + with closing(dbconn.connect(dbconn.DbURL())) as conn: + res = dbconn.query(conn, sql) + rows = res.fetchall() + except Exception as e: + raise Exception("Failed to query gp_stat_replication: %s" %str(e)) + + segments_with_running_basebackup = {row[0] for row in rows} + + if len(segments_with_running_basebackup) == 0: + logger.debug("No basebackup running") + return segments_with_running_basebackup class RecoveryTriplet: """ @@ -142,7 +167,15 @@ def _convert_requests_to_triplets(self, requests: List[RecoveryTripletRequest]) triplets = [] dbIdToPeerMap = self.gpArray.getDbIdToPeerMap() + + failed_segments_with_running_basebackup = [] + segments_with_running_basebackup = get_segments_with_running_basebackup() + for req in requests: + if req.failed.getSegmentContentId() in segments_with_running_basebackup: + failed_segments_with_running_basebackup.append(req.failed.getSegmentContentId()) + continue + # TODO: These 2 cases have different behavior which might be confusing to the user. # "|| ||" does full recovery # "||" does incremental recovery @@ -170,6 +203,11 @@ def _convert_requests_to_triplets(self, requests: List[RecoveryTripletRequest]) triplets.append(RecoveryTriplet(req.failed, peer, failover)) + if len(failed_segments_with_running_basebackup) > 0: + logger.warning( + "Found pg_basebackup running for segments with contentIds %s, skipping recovery of these segments" % ( + failed_segments_with_running_basebackup)) + return triplets diff --git a/gpMgmt/bin/gppylib/programs/clsSystemState.py b/gpMgmt/bin/gppylib/programs/clsSystemState.py index afbc8c01dcf..3874a875436 100644 --- a/gpMgmt/bin/gppylib/programs/clsSystemState.py +++ b/gpMgmt/bin/gppylib/programs/clsSystemState.py @@ -9,7 +9,7 @@ from optparse import OptionGroup import sys import collections -import pgdb +import psycopg2 from contextlib import closing from gppylib import gparray, gplog @@ -969,7 +969,7 @@ def _get_unsync_segs_add_wal_remaining_bytes(data, gpArray): wal_sync_bytes_out = 'Unknown' unsync_segs.append(s) data.addValue(VALUE__REPL_SYNC_REMAINING_BYTES, wal_sync_bytes_out) - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( s.dbid, s.hostname, s.port )) @@ -1041,7 +1041,7 @@ def _add_replication_info(data, primary, mirror): cursor.close() - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( primary.dbid, primary.hostname, primary.port )) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py index 99e6af0ef7d..302c9f83ddc 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py @@ -11,7 +11,7 @@ class GpCheckCatTestCase(GpTestCase): def setUp(self): self.logger = Mock(spec=['log', 'info', 'debug', 'error']) - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor']) self.autoCast = {'regproc': '::oid', 'regprocedure': '::oid', 'regoper': '::oid', @@ -25,9 +25,10 @@ def setUp(self): self.full_join_cat_tables = set(['pg_attribute','gp_distribution_policy','pg_appendonly','pg_constraint','pg_index']) self.foreign_key_check= Mock(spec=['runCheck']) self.foreign_key_check.runCheck.return_value = [] - self.db_connection.query.return_value.ntuples.return_value = 2 - self.db_connection.query.return_value.listfields.return_value = ['pkey1', 'pkey2'] - self.db_connection.query.return_value.getresult.return_value = [('r1','r2'), ('r3','r4')] + + self.db_connection.cursor.return_value.rowcount = 2 + self.db_connection.cursor.return_value.description = [('pkey1',), ('pkey2',)] + self.db_connection.cursor.return_value.fetchall.return_value = [('r1','r2'), ('r3','r4')] def test_get_fk_query_left_join_returns_the_correct_query(self): @@ -127,7 +128,7 @@ def test_checkTableForeignKey__returns_correct_join_query(self, log_literal_mock self.assertEqual(len(issue_list) , 2) self.assertEqual(issue_list[0], ('pg_class', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) self.assertEqual(issue_list[1], ('arbitrary_catalog_table', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) - self.assertEqual(self.db_connection.query.call_count, 2) + self.assertEqual(self.db_connection.cursor.call_count, 2) def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter=True): if with_filter: @@ -168,7 +169,7 @@ def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter= self.assertEqual(fk_query_full_join_mock.call_count, 0) fk_query_left_join_mock.assert_has_calls(foreign_key_mock_calls_left, any_order=False) - self.db_connection.query.call_count = 0 + self.db_connection.cursor.call_count = 0 fk_query_full_join_mock.call_count = 0 fk_query_left_join_mock.call_count = 0 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gparray.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gparray.py index d7d92ec036c..21d7351c5a5 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gparray.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gparray.py @@ -235,6 +235,127 @@ def test_initFromCatalog_mismatched_versions(self, mock_connect, mock_query): with self.assertRaisesRegex(Exception, 'Cannot connect to GPDB version 5 from installed version 7'): GpArray.initFromCatalog(None) + def test_get_min_primary_port_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = 6000 + actual = GpArray.get_min_primary_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_min_primary_port_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = 6000 + actual = GpArray.get_min_primary_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_max_primary_port_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = 6001 + actual = GpArray.get_max_primary_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_max_primary_port_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = 6001 + actual = GpArray.get_max_primary_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_min_mirror_port_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = 7000 + actual = GpArray.get_min_mirror_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_min_mirror_port_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = 7000 + actual = GpArray.get_min_mirror_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_max_mirror_port_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = 7001 + actual = GpArray.get_max_mirror_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_max_mirror_port_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = 7001 + actual = GpArray.get_max_mirror_port(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_primary_port_list_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = [6000, 6001] + actual = GpArray.get_primary_port_list(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_primary_port_list_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = [6000, 6001] + actual = GpArray.get_primary_port_list(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_primary_port_list_len_exception(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + self.gpArray.segmentPairs = [] + with self.assertRaisesRegex(Exception, 'No primary ports found in array.'): + GpArray.get_primary_port_list(self.gpArray) + + def test_get_mirror_port_list_when_cluster_balanced(self): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + expected = [7000, 7001] + actual = GpArray.get_mirror_port_list(self.gpArray) + self.assertEqual(expected, actual) + + def test_get_mirror_port_list_when_cluster_not_balanced(self): + self.gpArray = self._createUnBalancedGpArrayWith2Primary2Mirrors() + expected = [7000, 7001] + actual = GpArray.get_mirror_port_list(self.gpArray) + self.assertEqual(expected, actual) + + @patch('gppylib.gparray.GpArray.get_mirroring_enabled', return_value=False) + def test_get_mirror_port_list_no_mirror_exception(self,mock1): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + with self.assertRaisesRegex(Exception, 'Mirroring is not enabled'): + GpArray.get_mirror_port_list(self.gpArray) + + @patch('gppylib.gparray.GpArray.get_mirroring_enabled', return_value=True) + def test_get_mirror_port_list_len_exception(self, mock1): + self.gpArray = self._createBalancedGpArrayWith2Primary2Mirrors() + self.gpArray.segmentPairs = [] + with self.assertRaisesRegex(Exception, 'No mirror ports found in array.'): + GpArray.get_mirror_port_list(self.gpArray) + + def _createBalancedGpArrayWith2Primary2Mirrors(self): + self.coordinator = Segment.initFromString( + "1|-1|p|p|s|u|cdw|cdw|6432|/data/coordinator") + self.primary0 = Segment.initFromString( + "2|0|p|p|s|u|sdw1|sdw1|6000|/data/primary0") + self.primary1 = Segment.initFromString( + "3|1|p|p|s|u|sdw2|sdw2|6001|/data/primary1") + self.mirror0 = Segment.initFromString( + "4|0|m|m|s|u|sdw2|sdw2|7000|/data/mirror0") + self.mirror1 = Segment.initFromString( + "5|1|m|m|s|u|sdw1|sdw1|7001|/data/mirror1") + self.standby = Segment.initFromString( + "6|-1|m|m|s|u|sdw3|sdw3|6432|/data/standby") + return GpArray([self.coordinator, self.standby, self.primary0, self.primary1, self.mirror0, self.mirror1]) + + def _createUnBalancedGpArrayWith2Primary2Mirrors(self): + self.coordinator = Segment.initFromString( + "1|-1|p|p|s|u|cdw|cdw|6432|/data/coordinator") + self.primary0 = Segment.initFromString( + "2|0|m|p|s|u|sdw1|sdw1|6000|/data/primary0") + self.primary1 = Segment.initFromString( + "3|1|m|p|s|u|sdw2|sdw2|6001|/data/primary1") + self.mirror0 = Segment.initFromString( + "4|0|p|m|s|u|sdw2|sdw2|7000|/data/mirror0") + self.mirror1 = Segment.initFromString( + "5|1|p|m|s|u|sdw1|sdw1|7001|/data/mirror1") + self.standby = Segment.initFromString( + "6|-1|m|m|s|u|sdw3|sdw3|6432|/data/standby") + return GpArray([self.coordinator, self.standby, self.primary0, self.primary1, self.mirror0, self.mirror1]) + def convert_bool(val): if type(val) is bool: return val diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py index 63f2d88fe89..3b476a3a0bb 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py @@ -18,11 +18,11 @@ def setUp(self): self.subject = imp.load_source('gpcheckcat', gpcheckcat_file) self.subject.check_gpexpand = lambda : (True, "") - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor', 'set_session']) self.unique_index_violation_check = Mock(spec=['runCheck']) self.foreign_key_check = Mock(spec=['runCheck', 'checkTableForeignKey']) self.apply_patches([ - patch("gpcheckcat.pg.connect", return_value=self.db_connection), + patch("gpcheckcat.connect", return_value=self.db_connection), patch("gpcheckcat.UniqueIndexViolationCheck", return_value=self.unique_index_violation_check), patch("gpcheckcat.ForeignKeyCheck", return_value=self.foreign_key_check), patch('os.environ', new={}), @@ -129,23 +129,26 @@ def test_drop_leaked_schemas__when_leaked_schemas_exist__reports_which_schemas_a self.assertIn(expected_message, log_messages) def test_automatic_thread_count(self): - self.db_connection.query.return_value.getresult.return_value = [[0]] + self.db_connection.cursor.return_value.fetchall.return_value = [[0]] self._run_batch_size_experiment(100) self._run_batch_size_experiment(101) + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gppylib.gplog.log_literal') - def test_truncate_batch_size(self, mock_log, mock_gpcheckcat, mock_sys_exit): + def test_truncate_batch_size(self, mock_log, mock_sys_exit, mock_gpcatalog, mock_version): self.subject.GV.opt['-B'] = 300 # override the setting from available memory # setup conditions for 50 primaries and plenty of RAM such that max threads > 50 primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries testargs = ['some_string','-port 1', '-R foo'] @@ -221,10 +224,11 @@ def test_checkForeignKey__no_arg(self, process_foreign_key_mock): self.foreign_key_check.runCheck.assert_called_once_with(cat_tables) # Test gpcheckat -C option with checkForeignKey + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gpcheckcat.checkTableMissingEntry') - def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): + def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3, mock4): self.subject.checkForeignKey = Mock() gpcat_class_mock = Mock(spec=['getCatalogTable']) cat_obj_mock = Mock() @@ -234,8 +238,12 @@ def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.GV.opt['-C'] = 'pg_class' @@ -314,7 +322,13 @@ def test_skip_one_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s test2'] @@ -330,7 +344,13 @@ def test_skip_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test1, test2"] @@ -346,7 +366,11 @@ def test_skip_test_warning(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test_invalid, test2"] @@ -365,7 +389,13 @@ def test_run_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-R', "test1, test2"] diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py index 54fa8f3cf7c..42afb96763d 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py @@ -5,11 +5,10 @@ import shutil import sys import tempfile - +from psycopg2 import DatabaseError from gppylib.gparray import Segment, GpArray, SegmentPair from gpconfig_modules.parse_guc_metadata import ParseGuc import errno -from pg import DatabaseError from .gp_unittest import * from unittest.mock import * diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand.py index 8cab75222f9..4dff31a4448 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpexpand.py @@ -9,6 +9,7 @@ from gppylib.gplog import * from gppylib.system.configurationInterface import GpConfigurationProvider from gppylib.system.environment import GpCoordinatorEnvironment +from gppylib.db import dbconn import io import sys @@ -71,8 +72,8 @@ def tearDown(self): sys.argv = self.old_sys_argv super(GpExpand, self).tearDown() - # @patch('gpexpand.PgControlData.return_value.get_value', side_effect=[1, 1, 0]) - def test_validate_heap_checksums_aborts_when_cluster_inconsistent(self): + @patch('gpexpand.is_cluster_up_and_balanced', return_value=True) + def test_validate_heap_checksums_aborts_when_cluster_inconsistent(self, mock1): self.options.filename = '/tmp/doesnotexist' # Replacement of the sys.argv self.mock_heap_checksum.return_value.get_segments_checksum_settings.return_value = ([1], [0]) @@ -94,8 +95,8 @@ def test_validate_heap_checksums_aborts_when_cluster_inconsistent(self): @patch('gpexpand.FileDirExists.return_value.filedir_exists', return_value=True) @patch('gpexpand.FileDirExists', return_value=Mock()) - # @patch('gpexpand.HeapChecksum.PgControlData.return_value.get_value', side_effect=[1, 1, 1]) - def test_validate_heap_checksums_completes_when_cluster_consistent(self, mock1, mock2): + @patch('gpexpand.is_cluster_up_and_balanced', return_value=True) + def test_validate_heap_checksums_completes_when_cluster_consistent(self, mock1, mock2, mock3): """ If all the segment checksums match the checksum at the coordinator, then the cluster is consistent. This is essentially making sure that the validate_heap_checksums() internal method has not detected any @@ -127,6 +128,33 @@ def test_nonstandard_gpArray_user_aborts(self): self.assertIn('The current system appears to be non-standard.', mock_stdout.getvalue()) self.subject.logger.info.assert_any_call("User Aborted. Exiting...") + + @patch('gppylib.db.dbconn.querySingleton', return_value=0) + def test_unit_cluster_up_and_balanced_true(self,mock1): + expected = True + actual = self.subject.is_cluster_up_and_balanced(dbconn.DbURL()) + self.assertEqual(actual, expected) + + @patch('gppylib.db.dbconn.querySingleton', return_value=2) + def test_unit_cluster_up_and_balanced_false(self, mock): + expected = False + actual = self.subject.is_cluster_up_and_balanced(dbconn.DbURL()) + self.assertEqual(actual, expected) + + @patch('gppylib.db.dbconn.querySingleton', side_effect=Exception()) + def test_unit_cluster_up_and_balanced_exception(self, mock1): + with self.assertRaises(Exception) as ex: + self.subject.is_cluster_up_and_balanced(dbconn.DbURL()) + + self.assertTrue('failed to query cluster role check' in str(ex.exception)) + + @patch('gppylib.db.dbconn.connect', side_effect=Exception()) + def test_unit_cluster_up_and_balanced_conn_exception(self, mock1): + with self.assertRaises(Exception) as ex: + self.subject.is_cluster_up_and_balanced(dbconn.DbURL()) + + self.assertTrue('failed to query cluster role check' in str(ex.exception)) + # # end tests for interview_setup() # diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py index e46fdd38b3b..42804faa845 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py @@ -1,6 +1,7 @@ import unittest import mock -import pgdb +import psycopg2 +import tempfile from gppylib import gparray from .gp_unittest import GpTestCase @@ -143,7 +144,7 @@ def test_add_replication_info_adds_unknowns_if_primary_is_down(self): @mock.patch('gppylib.db.dbconn.connect', autospec=True) def test_add_replication_info_adds_unknowns_if_connection_cannot_be_made(self, mock_connect): # Simulate a connection failure in dbconn.connect(). - mock_connect.side_effect = pgdb.InternalError('connection failure forced by unit test') + mock_connect.side_effect = psycopg2.InternalError('connection failure forced by unit test') GpSystemStateProgram._add_replication_info(self.data, self.primary, self.mirror) self.assertEqual('Unknown', self.data.getStrValue(self.mirror, VALUE__REPL_SENT_LSN)) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py index 1eda2e72a25..1cd81684c23 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py @@ -6,45 +6,57 @@ class LeakedSchemaDropperTestCase(GpTestCase): def setUp(self): - self.db_connection = Mock(spec=['query']) - - two_leaked_schemas = Mock() - two_leaked_schemas.getresult.return_value = [ + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.fetchall.return_value = [ ('fake_leak_1', 'something_else'), ('some"test"special_#;character--schema', 'something_else') ] - self.db_connection.query.return_value = two_leaked_schemas - self.subject = LeakedSchemaDropper() def test_drop_leaked_schemas__returns_a_list_of_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), ['fake_leak_1', 'some"test"special_#;character--schema']) def test_drop_leaked_schemas__when_there_are_no_leaked_schemas__returns_an_empty_list(self): - no_leaked_schemas = Mock() - no_leaked_schemas.getresult.return_value = [] - self.db_connection.query.return_value = no_leaked_schemas - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null_schema__returns_an_empty_list(self): - null_leaked_schema = Mock() - null_leaked_schema.getresult.return_value = [(None, 'something_else')] - self.db_connection.query.return_value = null_leaked_schema - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [(None, 'something_else')] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null__returns_an_empty_list(self): - self.db_connection.query.return_value = None - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__drops_orphaned_and_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.subject.drop_leaked_schemas(self.db_connection) drop_query_expected_list = [call("DROP SCHEMA IF EXISTS \"fake_leak_1\" CASCADE;"), call("DROP SCHEMA IF EXISTS \"some\"\"test\"\"special_#;character--schema\" CASCADE;")] - self.db_connection.query.assert_has_calls(drop_query_expected_list) + self.db_connection.cursor.return_value.__enter__.return_value.execute.assert_has_calls(drop_query_expected_list) if __name__ == '__main__': diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py index c9306d69b97..19ddb773232 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py @@ -9,33 +9,37 @@ def setUp(self): self.subject = UniqueIndexViolationCheck() self.index_query_result = Mock() - self.index_query_result.getresult.return_value = [ + self.index_query_result.fetchall.return_value = [ (9001, 'index1', 'table1', ['index1_column1','index1_column2']), (9001, 'index2', 'table1', ['index2_column1','index2_column2']) ] - self.violated_segments_query_result = Mock() - - self.db_connection = Mock(spec=['query']) - self.db_connection.query.side_effect = self.mock_query_return_value - - def mock_query_return_value(self, query_string): - if query_string == UniqueIndexViolationCheck.unique_indexes_query: - return self.index_query_result - else: - return self.violated_segments_query_result + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) def test_run_check__when_there_are_no_issues(self): - self.violated_segments_query_result.getresult.return_value = [] + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], + [], + [], + ] violations = self.subject.runCheck(self.db_connection) self.assertEqual(len(violations), 0) def test_run_check__when_index_is_violated(self): - self.violated_segments_query_result.getresult.side_effect = [ + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], [(-1,), (0,), (1,)], - [(-1,)] + [(-1,)], ] violations = self.subject.runCheck(self.db_connection) diff --git a/gpMgmt/bin/gppylib/utils.py b/gpMgmt/bin/gppylib/utils.py index fead818d3ac..42c77e620ab 100644 --- a/gpMgmt/bin/gppylib/utils.py +++ b/gpMgmt/bin/gppylib/utils.py @@ -5,8 +5,7 @@ from sys import * from xml.dom import minidom from xml.dom import Node - -import pgdb +import psycopg2 from gppylib.gplog import * logger = get_default_logger() @@ -503,14 +502,12 @@ def escapeDoubleQuoteInSQLString(string, forceDoubleQuote=True): string = '"' + string + '"' return string - -def Escape(query_str): - return pgdb.escape_string(query_str) - +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def escapeArrayElement(query_str): # also escape backslashes and double quotes, in addition to the doubling of single quotes - return pgdb.escape_string(query_str.encode(errors='backslashreplace')).decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') + return escape_string(query_str.encode(errors='backslashreplace')).encode().decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') # Transform Python list to Postgres array literal (of the form: '{...}') @@ -593,7 +590,7 @@ def formatInsertValuesList(row, starelid, inclHLL): # Format stavalues5 for an hll slot elif i == 30 and hll: if inclHLL: - val = '\'{"%s"}\'' % pgdb.escape_bytea(val[0]) + val = '\'{\\%s}\'' % val[0] rowVals.append('\t{0}::{1}'.format(val, 'bytea[]')) else: rowVals.append('\t{0}'.format('NULL::int4[]')) diff --git a/gpMgmt/bin/gpsd b/gpMgmt/bin/gpsd index ad61a1e3de2..c12c5eb85b7 100755 --- a/gpMgmt/bin/gpsd +++ b/gpMgmt/bin/gpsd @@ -12,8 +12,8 @@ import re from contextlib import closing from distutils.version import LooseVersion from optparse import OptionParser -import pgdb -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string +import psycopg2 gpsd_version = '%prog 1.0' @@ -39,6 +39,15 @@ def getVersion(envOpts): sys.exit(1) return cmd.communicate()[0].decode('ascii') +def get_num_segments(cursor): + query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" + try: + cursor.execute(query) + except psycopg2.DatabaseError as e: + sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') + sys.exit(1) + vals = cursor.fetchone() + return vals[0] def dumpSchema(connectionInfo, envOpts): dump_cmd = 'pg_dump -h {host} -p {port} -U {user} -s -x -O {database}'.format(**connectionInfo) @@ -74,7 +83,13 @@ def dumpTupleCount(cur): def dumpStats(cur, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgn.nspname NOT IN ' + \ sysnslist + \ @@ -93,7 +108,7 @@ def dumpStats(cur, inclHLL): cur.execute(query) for vals in ResultIter(cur): - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) print(pstring.format(vals[0], vals[2], ',\n'.join(rowVals))) @@ -142,6 +157,10 @@ def main(): 'database': db, 'options': pgoptions } + num_segments = 0 + with closing(psycopg2.connect(**connectionInfo)) as connection: + with closing(connection.cursor()) as cursor: + num_segments = get_num_segments(cursor) sys.stdout.writelines(['\n-- Greenplum database Statistics Dump', '\n-- Copyright (C) 2007 - 2014 Pivotal' '\n-- Database: ' + db, @@ -171,7 +190,7 @@ def main(): sys.stdout.flush() try: - with closing(pgdb.connect(**connectionInfo)) as connection: + with closing(psycopg2.connect(**connectionInfo)) as connection: with closing(connection.cursor()) as cursor: dumpTupleCount(cursor) dumpStats(cursor, inclHLL) @@ -180,11 +199,11 @@ def main(): 'which requires some data elements to be included in the output file.\n', 'Please review output file to ensure it is within corporate policy to transport the output file.\n']) - except pgdb.DatabaseError as err: # catch *all* exceptions + except psycopg2.DatabaseError as err: # catch *all* exceptions sys.stderr.write('Error while dumping statistics:\n') sys.stderr.write(str(err)) sys.exit(1) if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/bin/minirepro b/gpMgmt/bin/minirepro index 5342dcf175f..e8066ffd4d9 100755 --- a/gpMgmt/bin/minirepro +++ b/gpMgmt/bin/minirepro @@ -61,10 +61,10 @@ minirepro gptest -h locahost -U gpadmin -p 4444 -q ~/in.sql -f ~/out.sql import pwd import os, sys, re, json, platform, subprocess -import pgdb +import psycopg2 from optparse import OptionParser from datetime import datetime -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string version = '1.13' PATH_PREFIX = '/tmp/' @@ -97,11 +97,21 @@ def get_server_version(cursor): query = "select version()" try: cursor.execute(query) - vals = cursor.fetchone() - return vals[0] - except pgdb.DatabaseError as e: - sys.stderr.write('\nError while trying to find HAWQ/GPDB version.\n\n' + str(e) + '\n\n') + except psycopg2.DatabaseError as e: + sys.stderr.write('\nError while trying to find GPDB version.\n\n' + str(e) + '\n\n') sys.exit(1) + vals = cursor.fetchone() + return vals[0] + +def get_num_segments(cursor): + query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" + try: + cursor.execute(query) + except psycopg2.DatabaseError as e: + sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') + sys.exit(1) + vals = cursor.fetchone() + return vals[0] def parse_cmd_line(): p = OptionParser(usage='Usage: %prog [options]', version='%prog '+version, conflict_handler="resolve", epilog="WARNING: This tool collects statistics about your data, including most common values, which requires some data elements to be included in the output file. Please review output file to ensure it is within corporate policy to transport the output file.") @@ -126,7 +136,7 @@ def dump_query(connectionInfo, query_file): with open(query_file, 'r') as query_f: sql_text = query_f.read() - query = "select pg_catalog.gp_dump_query_oids('%s')" % Escape(sql_text) + query = "select pg_catalog.gp_dump_query_oids('%s')" % escape_string(sql_text) toolkit_sql = PATH_PREFIX + 'toolkit.sql' with open(toolkit_sql, 'w') as toolkit_f: @@ -177,7 +187,7 @@ def pg_dump_object(mr_query, connectionInfo, envOpts): out_file = PATH_PREFIX + PGDUMP_FILE dmp_cmd = 'pg_dump -h %s -p %s -U %s -sxO %s' % connectionInfo dmp_cmd = "%s --relation-oids %s --function-oids %s -f %s" % \ - (dmp_cmd, mr_query.relids, mr_query.funcids, Escape(out_file)) + (dmp_cmd, mr_query.relids, mr_query.funcids, escape_string(out_file)) print(dmp_cmd) p = subprocess.Popen(dmp_cmd, shell=True, stderr=subprocess.PIPE, env=envOpts) _, errormsg = p.communicate() @@ -203,11 +213,17 @@ def dump_tuple_count(cur, oid_str, f_out): for col, val, typ in zip(columns[2:], vals[2:], types): # i.e. relpages = 1::int, reltuples = 1.0::real lines.append('\t%s = %s::%s' % (col, val, typ)) - updateStmt = templateStmt.format(Escape(',\n'.join(lines)), Escape(vals[0]), Escape(vals[1])) + updateStmt = templateStmt.format(escape_string(',\n'.join(lines)), escape_string(vals[0]), escape_string(vals[1])) f_out.writelines(updateStmt) def dump_stats(cur, oid_str, f_out, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgc.oid in (%s) ' \ 'and pgn.nspname NOT LIKE \'pg_temp_%%\' ' \ @@ -229,7 +245,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): for vals in result_iter(cur): schemaname = vals[1] - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) # For non-catalog tables we don't need to delete stats first @@ -238,7 +254,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): if schemaname != 'pg_catalog': linecomment = '-- ' # This will comment out the DELETE query - f_out.writelines(pstring.format(Escape(vals[0]), Escape(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) + f_out.writelines(pstring.format(escape_string(vals[0]), escape_string(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) def main(): parser = parse_cmd_line() @@ -289,7 +305,7 @@ def main(): } print("Connecting to database: host=%s, port=%s, user=%s, db=%s ..." % connectionInfo) - conn = pgdb.connect(**connectionDict) + conn = psycopg2.connect(**connectionDict) cursor = conn.cursor() # get server version, which is dumped to minirepro output file @@ -332,7 +348,7 @@ def main(): # first create schema DDLs print("Writing schema DDLs ...") - table_schemas = ["CREATE SCHEMA %s;\n" % Escape(schema) for schema in mr_query.schemas if schema != 'public'] + table_schemas = ["CREATE SCHEMA %s;\n" % escape_string(schema) for schema in mr_query.schemas if schema != 'public'] f_out.writelines(table_schemas) # write relation and function DDLs @@ -377,4 +393,4 @@ def main(): print('Please review output file to ensure it is within corporate policy to transport the output file.') if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/sbin/gpsegstop.py b/gpMgmt/sbin/gpsegstop.py index d47fac68699..b9f53c19aa1 100755 --- a/gpMgmt/sbin/gpsegstop.py +++ b/gpMgmt/sbin/gpsegstop.py @@ -23,7 +23,6 @@ from gppylib.commands import gp from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, DEFAULT_SEGHOST_NUM_WORKERS from gppylib.commands import pg -from gppylib.db import dbconn from gppylib import pgconf from gppylib.commands.gp import is_pid_postmaster diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 7b346b04dde..4b825d36b22 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -19,7 +19,8 @@ from datetime import datetime, timedelta from os import path from contextlib import closing - +import psycopg2 +from psycopg2 import extras from gppylib.gparray import GpArray, ROLE_PRIMARY, ROLE_MIRROR from gppylib.commands.gp import SegmentStart, GpStandbyStart, CoordinatorStop from gppylib.commands import gp @@ -3095,31 +3096,41 @@ def impl(context, table_name): dbname = 'gptest' conn = dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False) context.long_run_select_only_conn = conn + cursor = conn.cursor() + context.long_run_select_only_cursor = cursor + + # Start a readonly transaction. + cursor.execute("BEGIN") query = """SELECT gp_segment_id, * from %s order by 1, 2""" % table_name - data_result = dbconn.query(conn, query).fetchall() + cursor.execute(query) + data_result = cursor.fetchall() + context.long_run_select_only_data_result = data_result query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] context.long_run_select_only_xid = xid @then('verify that long-run read-only transaction still exists on {table_name}') def impl(context, table_name): dbname = 'gptest' - conn = context.long_run_select_only_conn + cursor = context.long_run_select_only_cursor query = """SELECT gp_segment_id, * from %s order by 1, 2""" % table_name - data_result = dbconn.query(conn, query).fetchall() + cursor.execute(query) + data_result = cursor.fetchall() query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] if (xid != context.long_run_select_only_xid or data_result != context.long_run_select_only_data_result): error_str = "Incorrect xid or select result of long run read-only transaction: \ - xid(before %s, after %), result(before %s, after %s)" - raise Exception(error_str % (context.long_run_select_only_xid, xid, context.long_run_select_only_data_result, data_result)) + xid(before {}, after {}), result(before {}, after {})" + raise Exception(error_str.format(context.long_run_select_only_xid, xid, context.long_run_select_only_data_result, data_result)) @given('a long-run transaction starts') def impl(context): @@ -3127,30 +3138,36 @@ def impl(context): conn = dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False) context.long_run_conn = conn + cursor = conn.cursor() + context.long_run_cursor = cursor + + cursor.execute("BEGIN") + query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] context.long_run_xid = xid @then('verify that long-run transaction aborted for changing the catalog by creating table {table_name}') def impl(context, table_name): - dbname = 'gptest' - conn = context.long_run_conn + cursor = context.long_run_cursor query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] if context.long_run_xid != xid: raise Exception("Incorrect xid of long run transaction: before %s, after %s" % (context.long_run_xid, xid)); query = """CREATE TABLE %s (a INT)""" % table_name try: - data_result = dbconn.query(conn, query) - except Exception as msg: - key_msg = "FATAL: cluster is expanded" - if key_msg not in msg.__str__(): - raise Exception("transaction not abort correctly, errmsg:%s" % msg) + cursor.execute(query) + except Exception as e: + key_msg = "cluster is expanded from" + if key_msg not in str(e): + raise Exception("transaction not abort correctly, errmsg:%s" % str(e)) else: - raise Exception("transaction not abort, result:%s" % data_result) + raise Exception("transaction not abort") @when('verify that the cluster has {num_of_segments} new segments') @then('verify that the cluster has {num_of_segments} new segments') @@ -3753,7 +3770,7 @@ def impl(context): @then('the database locales are saved') def impl(context): - with closing(dbconn.connect(dbconn.DbURL())) as conn: + with closing(dbconn.connect(dbconn.DbURL(), cursorFactory=psycopg2.extras.NamedTupleCursor)) as conn: rows = dbconn.query(conn, "SELECT name, setting FROM pg_settings WHERE name LIKE 'lc_%'").fetchall() context.database_locales = {row.name: row.setting for row in rows} diff --git a/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py index aa9b4a011c1..1c20e690d53 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py @@ -28,7 +28,7 @@ def create_cluster(context, with_mirrors=True): cd ../gpAux/gpdemo; \ export DEMO_PORT_BASE={port_base} && \ export NUM_PRIMARY_MIRROR_PAIRS={num_primary_mirror_pairs} && \ - export WITH_MIRRORS={with_mirrors} && \A + export WITH_MIRRORS={with_mirrors} && \ ./demo_cluster.sh -d && ./demo_cluster.sh -c && \ ./demo_cluster.sh """.format(port_base=os.getenv('PORT_BASE', 15432), @@ -108,18 +108,17 @@ def step_impl(context): def step_impl(context): result_cursor = query_sql( "postgres", - "select pg_get_replication_slots() from gp_dist_random('gp_id') order by gp_segment_id" + "select (pg_get_replication_slots()).* from gp_dist_random('gp_id') order by gp_segment_id" ) if result_cursor.rowcount != context.current_cluster_size: raise Exception("expected all %d primaries to have replication slots, only %d have slots" % (context.current_cluster_size, results.rowcount)) - for content_id, result in enumerate(result_cursor.fetchall()): - pg_rep_slot = result[0] - if (pg_rep_slot[0], pg_rep_slot[2], pg_rep_slot[4]) != ('internal_wal_replication_slot','physical','f') : + for content_id, pg_rep_slot in enumerate(result_cursor.fetchall()): + if (pg_rep_slot[0], pg_rep_slot[2], pg_rep_slot[4]) != ('internal_wal_replication_slot', 'physical', False) : raise Exception( "expected replication slot to be active for content id %d, got %s" % - (content_id, result[0]) + (content_id, pg_rep_slot) ) @then('the mirrors should not have replication slots') diff --git a/gpMgmt/test/behave_utils/gpexpand_dml.py b/gpMgmt/test/behave_utils/gpexpand_dml.py index 8658c7e58ff..55f0e37b34d 100755 --- a/gpMgmt/test/behave_utils/gpexpand_dml.py +++ b/gpMgmt/test/behave_utils/gpexpand_dml.py @@ -31,9 +31,15 @@ def __init__(self, dbname, dmltype): def run(self): conn = dbconn.connect(dbconn.DbURL(dbname=self.dbname), unsetSearchPath=False) + with conn.cursor() as cur: + cur.execute("BEGIN") + self.loop(conn) self.verify(conn) + with conn.cursor() as cur: + cur.execute("COMMIT") + conn.commit() conn.close() @@ -109,13 +115,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c1 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() - for i in range(0, self.counter): - if i != int(results[i][0]): - self.report_incorrect_result() - return + for i in range(self.counter): + if i != int(results[i][0]): + self.report_incorrect_result() + return class TestUpdate(TestDML): datasize = 1000 @@ -135,13 +143,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c2 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() - for i in range(0, self.datasize): - if i + self.counter - 1 != int(results[i][0]): - self.report_incorrect_result() - return + for i in range(self.datasize): + if i + self.counter - 1 != int(results[i][0]): + self.report_incorrect_result() + return class TestDelete(TestDML): datasize = 100000 @@ -161,13 +171,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c1 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() - - for i in range(self.counter, self.datasize): - if i != int(results[i - self.counter][0]): - self.report_incorrect_result() - return + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() + + for i in range(self.counter, self.datasize): + if i != int(results[i - self.counter][0]): + self.report_incorrect_result() + return # for test only if __name__ == '__main__': diff --git a/gpMgmt/test/behave_utils/utils.py b/gpMgmt/test/behave_utils/utils.py index 16db49655da..36814f4cb80 100644 --- a/gpMgmt/test/behave_utils/utils.py +++ b/gpMgmt/test/behave_utils/utils.py @@ -11,15 +11,13 @@ import subprocess import difflib -import pg - from contextlib import closing from datetime import datetime from gppylib.commands.base import Command, ExecutionError, REMOTE from gppylib.commands.gp import chk_local_db_running, get_coordinatordatadir from gppylib.db import dbconn from gppylib.gparray import GpArray, MODE_SYNCHRONIZED - +from gppylib.utils import escape_string PARTITION_START_DATE = '2010-01-01' PARTITION_END_DATE = '2013-01-01' @@ -319,14 +317,14 @@ def check_table_exists(context, dbname, table_name, table_type=None, host=None, FROM pg_class c, pg_namespace n WHERE c.relname = '%s' AND n.nspname = '%s' AND c.relnamespace = n.oid; """ - SQL = SQL_format % (escape_string(tablename, conn=conn), escape_string(schemaname, conn=conn)) + SQL = SQL_format % (escape_string(tablename), escape_string(schemaname)) else: SQL_format = """ SELECT oid, relkind, relam, reloptions \ FROM pg_class \ WHERE relname = E'%s';\ """ - SQL = SQL_format % (escape_string(table_name, conn=conn)) + SQL = SQL_format % (escape_string(table_name)) table_row = None try: @@ -758,11 +756,6 @@ def replace_special_char_env(str): str = str.replace("$%s" % var, os.environ[var]) return str - -def escape_string(string, conn): - return pg.DB(db=conn).escape_string(string) - - def wait_for_unblocked_transactions(context, num_retries=150): """ Tries once a second to successfully commit a transaction to the database diff --git a/gpcontrib/gp_replica_check/gp_replica_check.py b/gpcontrib/gp_replica_check/gp_replica_check.py index 3d5bccaca22..60f2dc5eab4 100755 --- a/gpcontrib/gp_replica_check/gp_replica_check.py +++ b/gpcontrib/gp_replica_check/gp_replica_check.py @@ -37,6 +37,28 @@ import subprocess import threading import pipes # for shell-quoting, pipes.quote() +import os +from collections import defaultdict +import psycopg2 + +def run_sql(sql, host=None, port=None, + dbname="postgres", is_query=True, + is_utility=False): + if host is None: + host = os.getenv("PGHOST") + if port is None: + port = int(os.getenv("PGPORT")) + opt = "-c gp_role=utility" if is_utility else None + try: + with psycopg2.connect(dbname=dbname, host=host, port=port, options=opt) as conn: + with conn.cursor() as cur: + cur.execute(sql) + if is_query: + resultList = cur.fetchall() + return resultList + except Exception as e: + print('Exception: %s while running query %s dbname = %s' % (e, sql, dbname)) + class ReplicaCheck(threading.Thread): def __init__(self, segrow, datname, relation_types): diff --git a/python-dependencies.txt b/python-dependencies.txt index 9e63afef528..941857c4a28 100644 --- a/python-dependencies.txt +++ b/python-dependencies.txt @@ -1,3 +1,4 @@ psutil==5.7.0 pygresql==5.2 pyyaml==5.3.1 +psycopg2==2.9.6 diff --git a/src/test/regress/expected/db_size_functions.out b/src/test/regress/expected/db_size_functions.out index cf876246837..a3b398ff4aa 100644 --- a/src/test/regress/expected/db_size_functions.out +++ b/src/test/regress/expected/db_size_functions.out @@ -313,3 +313,31 @@ select pg_relation_size(oid) between 3000000 and 5000000 from pg_class where rel t (1 row) +create table heapsizetest_size(a bigint); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +copy (select pg_relation_size(oid) from pg_class where relname = 'heapsizetest') to '/tmp/t_heapsizetest_size_xxx'; +copy heapsizetest_size from '/tmp/t_heapsizetest_size_xxx'; +select count(distinct a) from heapsizetest_size; + count +------- + 1 +(1 row) + +\! rm /tmp/t_heapsizetest_size_xxx +insert into heapsizetest_size +select sum(size) +from +( + select pg_relation_size(oid) + from gp_dist_random('pg_class') + where relname = 'heapsizetest' +) x(size); +-- both method should compute the same result +select count(distinct a) from heapsizetest_size; + count +------- + 1 +(1 row) + +drop table heapsizetest_size; diff --git a/src/test/regress/expected/subselect_gp.out b/src/test/regress/expected/subselect_gp.out index 7c7bedb7865..f4bf661b303 100644 --- a/src/test/regress/expected/subselect_gp.out +++ b/src/test/regress/expected/subselect_gp.out @@ -2982,3 +2982,169 @@ select * from issue_12656 where (i, j) in (select distinct on (i) i, j from issu 1 | 10002 (1 row) +--- +--- Test param info is preserved when bringing a path to OuterQuery locus +--- +drop table if exists param_t; +NOTICE: table "param_t" does not exist, skipping +create table param_t (i int, j int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into param_t select i, i from generate_series(1,10)i; +analyze param_t; +explain (costs off) +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); + QUERY PLAN +----------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on param_t a + Filter: (SubPlan 1) + SubPlan 1 + -> Aggregate + -> Nested Loop + -> Nested Loop + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on param_t c + -> Materialize + -> Result + Filter: (d.i = a.i) + -> Limit + -> Result + Filter: (d.j = c.j) + -> Materialize + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on param_t d + -> Materialize + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on param_t b + Optimizer: Postgres query optimizer +(23 rows) + +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); + i | j +----+---- + 10 | 10 +(1 row) + +drop table if exists param_t; +-- A guard test case for gpexpand's populate SQL +-- Some simple notes and background is: we want to compute +-- table size efficiently, it is better to avoid invoke +-- pg_relation_size() in serial on QD, since this function +-- will dispatch for each tuple. The bad pattern SQL is like +-- select pg_relation_size(oid) from pg_class where xxx +-- The idea is force pg_relations_size is evaluated on each +-- segment and the sum the result together to get the final +-- result. To make sure correctness, we have to evaluate +-- pg_relation_size before any motion. The skill here is +-- to wrap this in a subquery, due to volatile of pg_relation_size, +-- this subquery won't be pulled up. Plus the skill of +-- gp_dist_random('pg_class') we can achieve this goal. +-- the below test is to verify the plan, we should see pg_relation_size +-- is evaludated on each segment and then motion then sum together. The +-- SQL pattern is a catalog join a table size "dict". +set gp_enable_multiphase_agg = on; +-- force nestloop join to make test stable since we +-- are testing plan and do not care about where we +-- put hash table. +set enable_hashjoin = off; +set enable_nestloop = on; +set enable_indexscan = off; +set enable_bitmapscan = off; +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> Nested Loop + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + Join Filter: (pc.oid = pg_class.oid) + -> Redistribute Motion 1:3 (slice2) + Output: pc.relname, pc.oid + Hash Key: pc.oid + -> Seq Scan on pg_catalog.pg_class pc + Output: pc.relname, pc.oid + -> Materialize + Output: pg_class.oid, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> HashAggregate + Output: pg_class.oid, sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text))) + Group Key: pg_class.oid + -> Redistribute Motion 3:3 (slice3; segments: 3) + Output: pg_class.oid, (pg_relation_size((pg_class.oid)::regclass, 'main'::text)) + Hash Key: pg_class.oid + -> Seq Scan on pg_catalog.pg_class + Output: pg_class.oid, pg_relation_size((pg_class.oid)::regclass, 'main'::text) + Optimizer: Postgres query optimizer +(21 rows) + +set gp_enable_multiphase_agg = off; +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> Nested Loop + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + Join Filter: (pc.oid = pg_class.oid) + -> Redistribute Motion 1:3 (slice2) + Output: pc.relname, pc.oid + Hash Key: pc.oid + -> Seq Scan on pg_catalog.pg_class pc + Output: pc.relname, pc.oid + -> Materialize + Output: pg_class.oid, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> HashAggregate + Output: pg_class.oid, sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text))) + Group Key: pg_class.oid + -> Redistribute Motion 3:3 (slice3; segments: 3) + Output: pg_class.oid, (pg_relation_size((pg_class.oid)::regclass, 'main'::text)) + Hash Key: pg_class.oid + -> Seq Scan on pg_catalog.pg_class + Output: pg_class.oid, pg_relation_size((pg_class.oid)::regclass, 'main'::text) + Optimizer: Postgres query optimizer +(21 rows) + +reset gp_enable_multiphase_agg; +reset enable_hashjoin; +reset enable_nestloop; +reset enable_indexscan; +reset enable_bitmapscan; diff --git a/src/test/regress/expected/subselect_gp_optimizer.out b/src/test/regress/expected/subselect_gp_optimizer.out index 5991016c98d..0b78e37d217 100644 --- a/src/test/regress/expected/subselect_gp_optimizer.out +++ b/src/test/regress/expected/subselect_gp_optimizer.out @@ -3072,3 +3072,169 @@ select * from issue_12656 where (i, j) in (select distinct on (i) i, j from issu 1 | 10002 (1 row) +--- +--- Test param info is preserved when bringing a path to OuterQuery locus +--- +drop table if exists param_t; +NOTICE: table "param_t" does not exist, skipping +create table param_t (i int, j int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into param_t select i, i from generate_series(1,10)i; +analyze param_t; +explain (costs off) +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); + QUERY PLAN +----------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + -> Seq Scan on param_t a + Filter: (SubPlan 1) + SubPlan 1 + -> Aggregate + -> Nested Loop + -> Nested Loop + -> Materialize + -> Broadcast Motion 3:3 (slice2; segments: 3) + -> Seq Scan on param_t c + -> Materialize + -> Result + Filter: (d.i = a.i) + -> Limit + -> Result + Filter: (d.j = c.j) + -> Materialize + -> Broadcast Motion 3:3 (slice3; segments: 3) + -> Seq Scan on param_t d + -> Materialize + -> Broadcast Motion 3:3 (slice4; segments: 3) + -> Seq Scan on param_t b + Optimizer: Postgres query optimizer +(23 rows) + +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); + i | j +----+---- + 10 | 10 +(1 row) + +drop table if exists param_t; +-- A guard test case for gpexpand's populate SQL +-- Some simple notes and background is: we want to compute +-- table size efficiently, it is better to avoid invoke +-- pg_relation_size() in serial on QD, since this function +-- will dispatch for each tuple. The bad pattern SQL is like +-- select pg_relation_size(oid) from pg_class where xxx +-- The idea is force pg_relations_size is evaluated on each +-- segment and the sum the result together to get the final +-- result. To make sure correctness, we have to evaluate +-- pg_relation_size before any motion. The skill here is +-- to wrap this in a subquery, due to volatile of pg_relation_size, +-- this subquery won't be pulled up. Plus the skill of +-- gp_dist_random('pg_class') we can achieve this goal. +-- the below test is to verify the plan, we should see pg_relation_size +-- is evaludated on each segment and then motion then sum together. The +-- SQL pattern is a catalog join a table size "dict". +set gp_enable_multiphase_agg = on; +-- force nestloop join to make test stable since we +-- are testing plan and do not care about where we +-- put hash table. +set enable_hashjoin = off; +set enable_nestloop = on; +set enable_indexscan = off; +set enable_bitmapscan = off; +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> Nested Loop + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + Join Filter: (pc.oid = pg_class.oid) + -> Redistribute Motion 1:3 (slice2) + Output: pc.relname, pc.oid + Hash Key: pc.oid + -> Seq Scan on pg_catalog.pg_class pc + Output: pc.relname, pc.oid + -> Materialize + Output: pg_class.oid, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> HashAggregate + Output: pg_class.oid, sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text))) + Group Key: pg_class.oid + -> Redistribute Motion 3:3 (slice3; segments: 3) + Output: pg_class.oid, (pg_relation_size((pg_class.oid)::regclass, 'main'::text)) + Hash Key: pg_class.oid + -> Seq Scan on pg_catalog.pg_class + Output: pg_class.oid, pg_relation_size((pg_class.oid)::regclass, 'main'::text) + Optimizer: Postgres query optimizer +(21 rows) + +set gp_enable_multiphase_agg = off; +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> Nested Loop + Output: pc.relname, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + Join Filter: (pc.oid = pg_class.oid) + -> Redistribute Motion 1:3 (slice2) + Output: pc.relname, pc.oid + Hash Key: pc.oid + -> Seq Scan on pg_catalog.pg_class pc + Output: pc.relname, pc.oid + -> Materialize + Output: pg_class.oid, (sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text)))) + -> HashAggregate + Output: pg_class.oid, sum((pg_relation_size((pg_class.oid)::regclass, 'main'::text))) + Group Key: pg_class.oid + -> Redistribute Motion 3:3 (slice3; segments: 3) + Output: pg_class.oid, (pg_relation_size((pg_class.oid)::regclass, 'main'::text)) + Hash Key: pg_class.oid + -> Seq Scan on pg_catalog.pg_class + Output: pg_class.oid, pg_relation_size((pg_class.oid)::regclass, 'main'::text) + Optimizer: Postgres query optimizer +(21 rows) + +reset gp_enable_multiphase_agg; +reset enable_hashjoin; +reset enable_nestloop; +reset enable_indexscan; +reset enable_bitmapscan; diff --git a/src/test/regress/sql/db_size_functions.sql b/src/test/regress/sql/db_size_functions.sql index 50dcc8beefa..23fffc1eb8b 100644 --- a/src/test/regress/sql/db_size_functions.sql +++ b/src/test/regress/sql/db_size_functions.sql @@ -100,3 +100,27 @@ select pg_total_relation_size('aocssizetest') = pg_table_size('aocssizetest'); -- plausible difference to the above scenarios would be that the function -- might get executed on different nodes, for example.) select pg_relation_size(oid) between 3000000 and 5000000 from pg_class where relname = 'heapsizetest'; -- 3637248 + +create table heapsizetest_size(a bigint); + +copy (select pg_relation_size(oid) from pg_class where relname = 'heapsizetest') to '/tmp/t_heapsizetest_size_xxx'; +copy heapsizetest_size from '/tmp/t_heapsizetest_size_xxx'; + +select count(distinct a) from heapsizetest_size; + +\! rm /tmp/t_heapsizetest_size_xxx + +insert into heapsizetest_size +select sum(size) +from +( + select pg_relation_size(oid) + from gp_dist_random('pg_class') + where relname = 'heapsizetest' +) x(size); + +-- both method should compute the same result +select count(distinct a) from heapsizetest_size; + +drop table heapsizetest_size; + diff --git a/src/test/regress/sql/subselect_gp.sql b/src/test/regress/sql/subselect_gp.sql index 8e36f03bf93..147f529b4fd 100644 --- a/src/test/regress/sql/subselect_gp.sql +++ b/src/test/regress/sql/subselect_gp.sql @@ -1190,3 +1190,96 @@ explain (costs off, verbose) select * from issue_12656 where (i, j) in (select distinct on (i) i, j from issue_12656 order by i, j desc); select * from issue_12656 where (i, j) in (select distinct on (i) i, j from issue_12656 order by i, j desc); + +--- +--- Test param info is preserved when bringing a path to OuterQuery locus +--- +drop table if exists param_t; + +create table param_t (i int, j int); +insert into param_t select i, i from generate_series(1,10)i; +analyze param_t; + +explain (costs off) +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); +select * from param_t a where a.i in + (select count(b.j) from param_t b, param_t c, + lateral (select * from param_t d where d.j = c.j limit 10) s + where s.i = a.i + ); + + +drop table if exists param_t; + +-- A guard test case for gpexpand's populate SQL +-- Some simple notes and background is: we want to compute +-- table size efficiently, it is better to avoid invoke +-- pg_relation_size() in serial on QD, since this function +-- will dispatch for each tuple. The bad pattern SQL is like +-- select pg_relation_size(oid) from pg_class where xxx +-- The idea is force pg_relations_size is evaluated on each +-- segment and the sum the result together to get the final +-- result. To make sure correctness, we have to evaluate +-- pg_relation_size before any motion. The skill here is +-- to wrap this in a subquery, due to volatile of pg_relation_size, +-- this subquery won't be pulled up. Plus the skill of +-- gp_dist_random('pg_class') we can achieve this goal. + +-- the below test is to verify the plan, we should see pg_relation_size +-- is evaludated on each segment and then motion then sum together. The +-- SQL pattern is a catalog join a table size "dict". + +set gp_enable_multiphase_agg = on; +-- force nestloop join to make test stable since we +-- are testing plan and do not care about where we +-- put hash table. +set enable_hashjoin = off; +set enable_nestloop = on; +set enable_indexscan = off; +set enable_bitmapscan = off; + +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + +set gp_enable_multiphase_agg = off; + +explain (verbose on, costs off) +with cte(table_oid, size) as +( + select + table_oid, + sum(size) size + from ( + select oid, + pg_relation_size(oid) + from gp_dist_random('pg_class') + ) x(table_oid, size) + group by table_oid +) +select pc.relname, ts.size +from pg_class pc, cte ts +where pc.oid = ts.table_oid; + +reset gp_enable_multiphase_agg; +reset enable_hashjoin; +reset enable_nestloop; +reset enable_indexscan; +reset enable_bitmapscan;