From 18e0d0a13460e3df1aab857cc5b5ab921a620839 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 22:58:51 +0900 Subject: [PATCH 01/34] Fix PYL-W0612: Unused variable found --- qds_sdk/commands.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 0966a9e2..08b810a7 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -103,7 +103,7 @@ def listparse(cls, args): (options, args) = cls.listparser.parse_args(args) except OptionParsingError as e: raise ParseError(e.msg, cls.listparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None return vars(options) @@ -394,7 +394,7 @@ def parse(cls, args): cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.script_location is not None: @@ -479,7 +479,7 @@ def parse(cls, args): cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.script_location is not None: @@ -633,7 +633,7 @@ def validate_script_location(cls, options): str(e), cls.optparser.format_help()) - + options.script_location = None if options.language == "sql": options.sql = q @@ -661,7 +661,7 @@ def parse(cls, args): (options, args) = cls.optparser.parse_args(args) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None SparkCommand.validate_program(options) @@ -733,7 +733,7 @@ def parse(cls, args): cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.script_location is not None: @@ -810,7 +810,7 @@ def parse(cls, args): (options, args) = cls.optparser.parse_args(args) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None parsed['label'] = options.label @@ -895,7 +895,7 @@ def parse(cls, args): cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.script_location is not None: @@ -993,7 +993,7 @@ def parse(cls, args): cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.script_location is not None: @@ -1146,7 +1146,7 @@ def parse(cls, args): except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None v = vars(options) @@ -1231,7 +1231,7 @@ def parse(cls, args): except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None v = vars(options) @@ -1338,7 +1338,7 @@ def parse(cls, args): except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None if options.macros is not None: @@ -1430,7 +1430,7 @@ def parse(cls, args): raise ParseError(msg, cls.optparser.format_help()) except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) - except OptionParsingExit as e: + except OptionParsingExit: return None params = vars(options) @@ -1526,7 +1526,7 @@ def _callback(downloaded, total): progress = downloaded*100/total sys.stderr.write('\r[{0}] {1}%'.format('#'*progress, progress)) sys.stderr.flush() - + m = _URI_RE.match(s3_path) bucket_name = m.group(1) bucket = boto_conn.get_bucket(bucket_name) From bc31ade1d03e161041ffc2876ab0a837ac515a42 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:06:26 +0900 Subject: [PATCH 02/34] skipcq PYL-W0402 --- qds_sdk/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 08b810a7..c7a949c3 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -10,14 +10,14 @@ from qds_sdk.exception import ParseError from qds_sdk.account import Account from qds_sdk.util import GentleOptionParser, OptionParsingError, OptionParsingExit, _is_cloud_url -from optparse import SUPPRESS_HELP +from optparse import SUPPRESS_HELP #skipcq: PYL-W0402 import boto import time import logging import sys import re -import pipes +import pipes #skipcq: PYL-W0402 import os import json import signal From 886462313ef5613b647139f880cd99e006cda6c8 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:06:44 +0900 Subject: [PATCH 03/34] Fix PYL-W1505: Method used has been deprecated --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index c7a949c3..7088c7c6 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1550,7 +1550,7 @@ def _callback(downloaded, total): if (e.status == 403): # SDK-191, boto gives an error while fetching the objects using versions which happens by default # in the get_contents_to_file() api. So attempt one without specifying version. - log.warn("Access denied while fetching the s3 object. Retrying without specifying the version....") + log.warning("Access denied while fetching the s3 object. Retrying without specifying the version....") key_instance.open() fp.write(key_instance.read()) key_instance.close() From 909e168eb2b68428e63d6071c74cf0b664f67d20 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:09:12 +0900 Subject: [PATCH 04/34] Fix PYL-R1714: Consider using "in" --- qds_sdk/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 7088c7c6..460fd1ef 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -64,7 +64,7 @@ def is_done(status): Returns: True/False """ - return status == "cancelled" or status == "done" or status == "error" + return status in ("cancelled", "done", "error") @staticmethod def is_success(status): @@ -1521,7 +1521,7 @@ def _callback(downloaded, total): `total`: Total file size to be downloaded (int) ''' - if (total is 0) or (downloaded == total): + if total in (0, downloaded): return progress = downloaded*100/total sys.stderr.write('\r[{0}] {1}%'.format('#'*progress, progress)) From d9ef87a282621b4342725585e4dc353214435ee6 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:11:03 +0900 Subject: [PATCH 05/34] Fix PYL-W1201: Logging is not lazy --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 460fd1ef..b6174478 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -308,7 +308,7 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N aws_secret_access_key=storage_credentials['storage_secret_key'], security_token=storage_credentials['session_token'], host=host) - log.info("Starting download from result locations: [%s]" % ",".join(r['result_location'])) + log.info("Starting download from result locations: [%s]", ",".join(r['result_location'])) # fetch latest value of num_result_dir num_result_dir = Command.find(self.id).num_result_dir @@ -1542,7 +1542,7 @@ def _callback(downloaded, total): key_instance = bucket.get_key(key_name) if key_instance is None: raise Exception("Results file not available on s3 yet. This can be because of s3 eventual consistency issues.") - log.info("Downloading file from %s" % s3_path) + log.info("Downloading file from %s", s3_path) if delim is None: try: key_instance.get_contents_to_file(fp) # cb=_callback @@ -1571,7 +1571,7 @@ def _callback(downloaded, total): if name.endswith('$folder$'): continue - log.info("Downloading file from %s" % name) + log.info("Downloading file from %s", name) if delim is None: one_path.get_contents_to_file(fp) # cb=_callback else: From a1b7f635dac6e1ad0d5c1e88afb00de210cf8604 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:12:50 +0900 Subject: [PATCH 06/34] Fix PYL-R1722: Use of "exit()" or "quit()" detected --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index b6174478..de13ed91 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -154,7 +154,7 @@ def run(cls, **kwargs): if sighandler.received_term_signal: logging.warning("Received signal {}. Canceling Qubole Command ID: {}".format(sighandler.last_signal, cmd.id)) cls.cancel(cmd) - exit() + sys.exit() time.sleep(Qubole.poll_interval) cmd = cls.find(cmd.id) if print_logs_live is True: From 95c313bbd56fc7f6eec2ae3c1c83116feca48960 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:14:14 +0900 Subject: [PATCH 07/34] Fix PTC-W0048: "if" statements can be merged --- qds_sdk/commands.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index de13ed91..2482c6a8 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -562,9 +562,8 @@ def validate_program(cls, options): # if both are true then atleast two option specified ==> raise ParseError if bool_program == bool_other_options: raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help()) - if bool_program: - if options.language is None: - raise ParseError("Unspecified language for Program", cls.optparser.format_help()) + if bool_program and options.language is None: + raise ParseError("Unspecified language for Program", cls.optparser.format_help()) @classmethod def validate_cmdline(cls, options): @@ -590,9 +589,8 @@ def validate_sql(cls, options): # if both are true then atleast two option specified => raise ParseError if bool_sql == bool_other_options: raise ParseError("Exactly One of script location or program or cmdline or sql or note_id should be specified", cls.optparser.format_help()) - if bool_sql: - if options.language is not None: - raise ParseError("Language cannot be specified with the 'sql' option", cls.optparser.format_help()) + if bool_sql and options.language is not None: + raise ParseError("Language cannot be specified with the 'sql' option", cls.optparser.format_help()) @classmethod def validate_script_location(cls, options): From e98794cc0a36df99f6960017cae53545c1054e02 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:20:21 +0900 Subject: [PATCH 08/34] skipcq PTC-W0065 --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 2482c6a8..325e5b72 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1511,7 +1511,7 @@ def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): `fp`: The file object where data is to be downloaded ''' #Progress bar to display download progress - def _callback(downloaded, total): + def _callback(downloaded, total): #skipcq PTC-W0065 ''' Call function for upload. From 3eb862f5da19cee80f6b10629834d6c8882bef5c Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:21:57 +0900 Subject: [PATCH 09/34] Fix FLK-E127: Continuation line over-indented for visual indent --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 325e5b72..4e4c1caa 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1189,7 +1189,7 @@ class DbImportCommand(Command): optparser.add_option("--name", dest="name", help="Assign a name to this command") optparser.add_option("--additional_options", - help="Additional Sqoop options which are needed enclose options in double or single quotes") + help="Additional Sqoop options which are needed enclose options in double or single quotes") optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", From 74f6d56186b22838c405725d1daf4413b681e1d8 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:25:39 +0900 Subject: [PATCH 10/34] Fix FLK-E231: Missing whitespace after ",", ";", or ":" --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 4e4c1caa..c9e08691 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -230,7 +230,7 @@ def get_log_partial(self, err_pointer=0, tmp_pointer=0): """ log_path = self.meta_data['logs_resource'] conn = Qubole.agent() - r = conn.get_raw(log_path, params={'err_file_processed':err_pointer, 'tmp_file_processed':tmp_pointer}) + r = conn.get_raw(log_path, params={'err_file_processed': err_pointer, 'tmp_file_processed': tmp_pointer}) if 'err_length' in r.headers.keys() and 'tmp_length' in r.headers.keys(): return [r.text, r.headers['err_length'], r.headers['tmp_length']] return [r.text, 0, 0] @@ -368,7 +368,7 @@ class HiveCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") @classmethod def parse(cls, args): @@ -510,7 +510,7 @@ def parse(cls, args): class SparkCommand(Command): usage = ("sparkcmd [options]") - allowedlanglist = ["python", "scala","R"] + allowedlanglist = ["python", "scala", "R"] optparser = GentleOptionParser(usage=usage) optparser.add_option("--program", dest="program",help=SUPPRESS_HELP) From e34d959bb5f3e4252c671c977e3523c27564682b Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:30:15 +0900 Subject: [PATCH 11/34] Fix PYL-R0123: Comparing to literal --- qds_sdk/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index c9e08691..8903cbf8 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1119,7 +1119,7 @@ def parse(cls, args): raise ParseError("dbtap_id and db_table are required", cls.optparser.format_help()) - if options.mode is "1": + if options.mode == "1": if options.hive_table is None: raise ParseError("hive_table is required for mode 1", cls.optparser.format_help()) @@ -1132,7 +1132,7 @@ def parse(cls, args): raise ParseError("db_update_mode should either be left blank for append " "mode or be 'updateonly' or 'allowinsert'", cls.optparser.format_help()) - if options.db_update_mode is "updateonly": + if options.db_update_mode == "updateonly": if options.db_update_keys is None: raise ParseError("db_update_keys is required when db_update_mode " "is 'updateonly'", From 56322c3b2653e7cfa02e249c1f113bb1df113b42 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:32:53 +0900 Subject: [PATCH 12/34] Fix FLK-E231: Missing whitespace after ",", ";", or ":" --- qds_sdk/commands.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 8903cbf8..70c96e16 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -513,7 +513,7 @@ class SparkCommand(Command): allowedlanglist = ["python", "scala", "R"] optparser = GentleOptionParser(usage=usage) - optparser.add_option("--program", dest="program",help=SUPPRESS_HELP) + optparser.add_option("--program", dest="program", help=SUPPRESS_HELP) optparser.add_option("--cmdline", dest="cmdline", help="command line for Spark") @@ -705,7 +705,7 @@ class PrestoCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") @classmethod def parse(cls, args): @@ -783,7 +783,7 @@ class HadoopCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") optparser.disable_interspersed_args() @@ -965,7 +965,7 @@ class PigCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", choices=[1,2,3], default=0, help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", choices=[1, 2, 3], default=0, help="Number of retries for a job") @classmethod def parse(cls, args): @@ -1091,7 +1091,7 @@ class DbExportCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") @classmethod def parse(cls, args): @@ -1194,7 +1194,7 @@ class DbImportCommand(Command): default=False, help="Fetch logs and print them to stderr.") optparser.add_option("--print-logs-live", action="store_true", dest="print_logs_live", default=False, help="Fetch logs and print them to stderr while command is running.") - optparser.add_option("--retry", dest="retry", default=0, choices=[1,2,3], help="Number of retries for a job") + optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") optparser.add_option("--partition_spec", dest="part_spec", default=None, help="Mode 1: (optional) Partition specification for Hive table") @@ -1485,7 +1485,7 @@ def _read_iteratively(key_instance, fp, delim): # Stream closes itself when the exception is raised return -def write_headers(qlog,fp): +def write_headers(qlog, fp): col_names = [] qlog = json.loads(qlog) if qlog["QBOL-QUERY-SCHEMA"] is not None: From ea64c13a7c6562fb8b0a714e09622f9e034e1e6c Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:35:38 +0900 Subject: [PATCH 13/34] Fix FLK-E251: Unexpected spaces around keyword / parameter equals --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 70c96e16..e221ed2c 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -532,7 +532,7 @@ class SparkCommand(Command): optparser.add_option("--cluster-label", dest="label", help="the label of the cluster to run the command on") - optparser.add_option("--language", dest="language", choices = allowedlanglist, help=SUPPRESS_HELP) + optparser.add_option("--language", dest="language", choices=allowedlanglist, help=SUPPRESS_HELP) optparser.add_option("--app-id", dest="app_id", type=int, help="The Spark Job Server app id to submit this snippet to.") @@ -543,9 +543,9 @@ class SparkCommand(Command): optparser.add_option("--pool", dest="pool", help="Specify the Fairscheduler pool name for the command to use") - optparser.add_option("--arguments", dest = "arguments", help = "Spark Submit Command Line Options") + optparser.add_option("--arguments", dest="arguments", help="Spark Submit Command Line Options") - optparser.add_option("--user_program_arguments", dest = "user_program_arguments", help = "Arguments for User Program") + optparser.add_option("--user_program_arguments", dest="user_program_arguments", help="Arguments for User Program") optparser.add_option("--print-logs", action="store_true", dest="print_logs", default=False, help="Fetch logs and print them to stderr.") From 4199b00d1d2a201af441bc7ded5506ee71ddae44 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:37:58 +0900 Subject: [PATCH 14/34] Fix FLK-E261: At least two spaces before inline comment --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index e221ed2c..55cd9c37 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -10,14 +10,14 @@ from qds_sdk.exception import ParseError from qds_sdk.account import Account from qds_sdk.util import GentleOptionParser, OptionParsingError, OptionParsingExit, _is_cloud_url -from optparse import SUPPRESS_HELP #skipcq: PYL-W0402 +from optparse import SUPPRESS_HELP #skipcq: PYL-W0402 import boto import time import logging import sys import re -import pipes #skipcq: PYL-W0402 +import pipes #skipcq: PYL-W0402 import os import json import signal @@ -144,7 +144,7 @@ def run(cls, **kwargs): # vars to keep track of actual logs bytes (err, tmp) and new bytes seen in each iteration err_pointer, tmp_pointer, new_bytes = 0, 0, 0 - print_logs_live = kwargs.pop("print_logs_live", None) # We don't want to send this to the API. + print_logs_live = kwargs.pop("print_logs_live", None) # We don't want to send this to the API. cmd = cls.create(**kwargs) From 2c8b65d4d48989fc953e0356f60bddfd18e883cc Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:39:22 +0900 Subject: [PATCH 15/34] Fix FLK-E262: Inline comment should start with "#" --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 55cd9c37..3d59a5cf 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -10,14 +10,14 @@ from qds_sdk.exception import ParseError from qds_sdk.account import Account from qds_sdk.util import GentleOptionParser, OptionParsingError, OptionParsingExit, _is_cloud_url -from optparse import SUPPRESS_HELP #skipcq: PYL-W0402 +from optparse import SUPPRESS_HELP # skipcq: PYL-W0402 import boto import time import logging import sys import re -import pipes #skipcq: PYL-W0402 +import pipes # skipcq: PYL-W0402 import os import json import signal @@ -1511,7 +1511,7 @@ def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): `fp`: The file object where data is to be downloaded ''' #Progress bar to display download progress - def _callback(downloaded, total): #skipcq PTC-W0065 + def _callback(downloaded, total): # skipcq PTC-W0065 ''' Call function for upload. From 368ed54d25a9a3f07966c8d3acabef41e778cc1f Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:41:58 +0900 Subject: [PATCH 16/34] Fix FLK-E302: Expected 2 blank lines --- qds_sdk/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 3d59a5cf..a553cfff 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -422,6 +422,7 @@ def parse(cls, args): v["command_type"] = "HiveCommand" return v + class SqlCommand(Command): usage = ("sqlcmd [options]") @@ -507,6 +508,7 @@ def parse(cls, args): v["command_type"] = "SqlCommand" return v + class SparkCommand(Command): usage = ("sparkcmd [options]") @@ -1151,6 +1153,7 @@ def parse(cls, args): v["command_type"] = "DbExportCommand" return v + class DbImportCommand(Command): usage = "dbimportcmd [options]" From ecddf70f2121245046f8330041bac107024c4a6d Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:44:01 +0900 Subject: [PATCH 17/34] Fix FLK-E265: Block comment should start with "#" --- qds_sdk/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index a553cfff..028d44c5 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1513,7 +1513,7 @@ def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): `fp`: The file object where data is to be downloaded ''' - #Progress bar to display download progress + # Progress bar to display download progress def _callback(downloaded, total): # skipcq PTC-W0065 ''' Call function for upload. @@ -1533,7 +1533,7 @@ def _callback(downloaded, total): # skipcq PTC-W0065 bucket = boto_conn.get_bucket(bucket_name) retries = 6 if s3_path.endswith('/') is False: - #It is a file + # It is a file key_name = m.group(2) key_instance = bucket.get_key(key_name) while key_instance is None and retries > 0: @@ -1562,7 +1562,7 @@ def _callback(downloaded, total): # skipcq PTC-W0065 _read_iteratively(key_instance, fp, delim=delim) else: - #It is a folder + # It is a folder key_prefix = m.group(2) bucket_paths = bucket.list(key_prefix) for one_path in bucket_paths: From 63738d4585897ab788599ebb558c2df06d3bdecd Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:46:14 +0900 Subject: [PATCH 18/34] Fix FLK-E302: Expected 2 blank lines --- qds_sdk/commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 028d44c5..c9af5edc 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1438,6 +1438,7 @@ def parse(cls, args): params["command_type"] = "JupyterNotebookCommand" return params + class SignalHandler: """ Catch terminate signals to allow graceful termination of run() @@ -1468,6 +1469,7 @@ def validate_json_input(string, option_type, cls): raise ParseError("Given %s is not valid JSON: %s" % (option_type, str(e)), cls.optparser.format_help()) + def _read_iteratively(key_instance, fp, delim): key_instance.open_read() while True: @@ -1488,6 +1490,7 @@ def _read_iteratively(key_instance, fp, delim): # Stream closes itself when the exception is raised return + def write_headers(qlog, fp): col_names = [] qlog = json.loads(qlog) From ce1801ad30ec093a16ef4e99468a36038a9e1dd6 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:48:13 +0900 Subject: [PATCH 19/34] Fix FLK-E303: Too many blank lines found --- qds_sdk/commands.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index c9af5edc..89009cb0 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -251,7 +251,6 @@ def get_jobs_id(cls, id): r = conn.get_raw(cls.element_path(id) + "/jobs") return r.text - def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=None, arguments=[]): """ Fetches the result for the command represented by this object @@ -278,7 +277,6 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N if include_header not in ('true', 'false'): raise ParseError("incude_header can be either true or false") - r = conn.get(result_path, {'inline': inline, 'include_headers': include_header}) if r.get('inline'): raw_results = r['results'] @@ -329,7 +327,6 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N fp.write(",".join(r['result_location'])) - class HiveCommand(Command): usage = ("hivecmd [options]") @@ -633,7 +630,6 @@ def validate_script_location(cls, options): str(e), cls.optparser.format_help()) - options.script_location = None if options.language == "sql": options.sql = q @@ -641,7 +637,6 @@ def validate_script_location(cls, options): else: options.program = q - @classmethod def parse(cls, args): """ @@ -677,7 +672,6 @@ def parse(cls, args): return v - class PrestoCommand(Command): usage = ("prestocmd [options]") @@ -1201,7 +1195,6 @@ class DbImportCommand(Command): optparser.add_option("--partition_spec", dest="part_spec", default=None, help="Mode 1: (optional) Partition specification for Hive table") - @classmethod def parse(cls, args): """ From 574192a4159d98a296f5157c488c02329f92c8e3 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Sun, 15 Oct 2023 23:49:17 +0900 Subject: [PATCH 20/34] Fix FLK-E303: Too many blank lines found --- qds_sdk/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 89009cb0..fa1a36b6 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1194,7 +1194,6 @@ class DbImportCommand(Command): optparser.add_option("--retry", dest="retry", default=0, choices=[1, 2, 3], help="Number of retries for a job") optparser.add_option("--partition_spec", dest="part_spec", default=None, help="Mode 1: (optional) Partition specification for Hive table") - @classmethod def parse(cls, args): """ From c02e4e98e052bd81e520d76beaf15a275aa8f4f1 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:10:42 +0900 Subject: [PATCH 21/34] Fix FLK-E501: Line too long --- .deepsource.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.deepsource.toml b/.deepsource.toml index 39693798..294512a9 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -13,6 +13,9 @@ name = 'python' enabled = true runtime_version = '2.x.x' + [analyzers.meta] + max_line_length = 200 + # Test coverage analyzer [[analyzers]] name = "test-coverage" From 2e890ee552abb84762459b21c70bd4d696590d90 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:14:15 +0900 Subject: [PATCH 22/34] Fix FLK-D300: Use """triple double quotes""" --- qds_sdk/commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index fa1a36b6..269bd3d5 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1498,7 +1498,7 @@ def write_headers(qlog, fp): def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): - ''' + """ Downloads the contents of all objects in s3_path into fp Args: @@ -1507,16 +1507,16 @@ def _download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=None): `s3_path`: S3 path to be downloaded `fp`: The file object where data is to be downloaded - ''' + """ # Progress bar to display download progress def _callback(downloaded, total): # skipcq PTC-W0065 - ''' + """ Call function for upload. `downloaded`: File size already downloaded (int) `total`: Total file size to be downloaded (int) - ''' + """ if total in (0, downloaded): return progress = downloaded*100/total From bc57813b9723dbe2997679d18d365efb9b3b2d81 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:15:39 +0900 Subject: [PATCH 23/34] Fix PYL-C0325: Unnecessary parentheses after keyword --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 269bd3d5..82b3876f 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1543,7 +1543,7 @@ def _callback(downloaded, total): # skipcq PTC-W0065 try: key_instance.get_contents_to_file(fp) # cb=_callback except boto.exception.S3ResponseError as e: - if (e.status == 403): + if e.status == 403: # SDK-191, boto gives an error while fetching the objects using versions which happens by default # in the get_contents_to_file() api. So attempt one without specifying version. log.warning("Access denied while fetching the s3 object. Retrying without specifying the version....") From 2265a60267f49e719b142f5138fccfae960b629d Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:17:57 +0900 Subject: [PATCH 24/34] skiqcq PYL-W0511 --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 82b3876f..fd1c6c71 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -1220,7 +1220,7 @@ def parse(cls, args): raise ParseError("dbtap_id and db_table are required", cls.optparser.format_help()) - # TODO: Semantic checks for parameters in mode 1 and 2 + # TODO: Semantic checks for parameters in mode 1 and 2 # skipcq PYL-W0511 except OptionParsingError as e: raise ParseError(e.msg, cls.optparser.format_help()) From e85e51d4577c75843e7729cefccff0d0d5b3268b Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:19:17 +0900 Subject: [PATCH 25/34] Fix FLK-D200: One-line docstring should fit on one line with quotes --- qds_sdk/commands.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index fd1c6c71..568439da 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -188,9 +188,7 @@ def cancel_id(cls, id): return conn.put(cls.element_path(id), data) def cancel(self): - """ - Cancels command represented by this object - """ + """Cancels command represented by this object""" self.__class__.cancel_id(self.id) @classmethod @@ -1432,9 +1430,7 @@ def parse(cls, args): class SignalHandler: - """ - Catch terminate signals to allow graceful termination of run() - """ + """Catch terminate signals to allow graceful termination of run()""" def __init__(self): self.last_signal = None From fa2af398cec536da9c1ae43f4b211d3a99162874 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:21:47 +0900 Subject: [PATCH 26/34] skipcq PY-D0003 --- qds_sdk/commands.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 568439da..dccf8806 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -67,7 +67,7 @@ def is_done(status): return status in ("cancelled", "done", "error") @staticmethod - def is_success(status): + def is_success(status): # skipcq PY-D0003 return status == "done" @classmethod @@ -98,7 +98,7 @@ def list(cls, **kwargs): return conn.get(cls.rest_entity_path, params=params) @classmethod - def listparse(cls, args): + def listparse(cls, args): # skipcq PY-D0003 try: (options, args) = cls.listparser.parse_args(args) except OptionParsingError as e: @@ -551,7 +551,7 @@ class SparkCommand(Command): optparser.add_option("--retry", dest="retry", default=0, help="Number of retries") @classmethod - def validate_program(cls, options): + def validate_program(cls, options): # skipcq PY-D0003 bool_program = options.program is not None bool_other_options = options.script_location is not None or options.cmdline is not None or options.sql is not None or options.note_id is not None @@ -563,7 +563,7 @@ def validate_program(cls, options): raise ParseError("Unspecified language for Program", cls.optparser.format_help()) @classmethod - def validate_cmdline(cls, options): + def validate_cmdline(cls, options): # skipcq PY-D0003 bool_cmdline = options.cmdline is not None bool_other_options = options.script_location is not None or options.program is not None or options.sql is not None or options.note_id is not None @@ -578,7 +578,7 @@ def validate_cmdline(cls, options): raise ParseError("app_id cannot be specified with the commandline option", cls.optparser.format_help()) @classmethod - def validate_sql(cls, options): + def validate_sql(cls, options): # skipcq PY-D0003 bool_sql = options.sql is not None bool_other_options = options.script_location is not None or options.program is not None or options.cmdline is not None or options.note_id is not None @@ -590,7 +590,7 @@ def validate_sql(cls, options): raise ParseError("Language cannot be specified with the 'sql' option", cls.optparser.format_help()) @classmethod - def validate_script_location(cls, options): + def validate_script_location(cls, options): # skipcq PY-D0003 bool_script_location = options.script_location is not None bool_other_options = options.program is not None or options.cmdline is not None or options.sql is not None or options.note_id is not None @@ -1442,7 +1442,7 @@ def __init__(self): for signum in self.term_signals: signal.signal(signum, self.handler) - def handler(self, signum, frame): + def handler(self, signum, frame): # skipcq PY-D0003 self.last_signal = signum if signum in self.term_signals: self.received_term_signal = True @@ -1458,7 +1458,7 @@ def validate_json_input(string, option_type, cls): cls.optparser.format_help()) -def _read_iteratively(key_instance, fp, delim): +def _read_iteratively(key_instance, fp, delim): # skipcq PY-D0003 key_instance.open_read() while True: try: @@ -1479,7 +1479,7 @@ def _read_iteratively(key_instance, fp, delim): return -def write_headers(qlog, fp): +def write_headers(qlog, fp): # skipcq PY-D0003 col_names = [] qlog = json.loads(qlog) if qlog["QBOL-QUERY-SCHEMA"] is not None: From febd09226ecefd1155629652ee0a83ec7122f5cd Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:24:09 +0900 Subject: [PATCH 27/34] Fix PYL-W0105: Unassigned string statement --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index dccf8806..4873fade 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -35,7 +35,7 @@ class Command(Resource): commands can subclass this. """ - """all commands use the /commands endpoint""" + # all commands use the /commands endpoint rest_entity_path = "commands" listusage = " list [options]" From 3c6b523d7ac05d82e314febe574985a66ec53765 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:25:51 +0900 Subject: [PATCH 28/34] skipcq PY-D0002 --- qds_sdk/commands.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 4873fade..0ad08d4d 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -325,7 +325,7 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True, qlog=N fp.write(",".join(r['result_location'])) -class HiveCommand(Command): +class HiveCommand(Command): # skipcq PY-D0002 usage = ("hivecmd [options]") @@ -418,7 +418,7 @@ def parse(cls, args): return v -class SqlCommand(Command): +class SqlCommand(Command): # skipcq PY-D0002 usage = ("sqlcmd [options]") @@ -504,7 +504,7 @@ def parse(cls, args): return v -class SparkCommand(Command): +class SparkCommand(Command): # skipcq PY-D0002 usage = ("sparkcmd [options]") allowedlanglist = ["python", "scala", "R"] @@ -670,7 +670,7 @@ def parse(cls, args): return v -class PrestoCommand(Command): +class PrestoCommand(Command): # skipcq PY-D0002 usage = ("prestocmd [options]") @@ -753,7 +753,7 @@ def parse(cls, args): return v -class HadoopCommand(Command): +class HadoopCommand(Command): # skipcq PY-D0002 subcmdlist = ["jar", "s3distcp", "streaming"] usage = "hadoopcmd [options] <%s> [arg2] ..." % "|".join(subcmdlist) @@ -828,7 +828,7 @@ def parse(cls, args): return parsed -class ShellCommand(Command): +class ShellCommand(Command): # skipcq PY-D0002 usage = ("shellcmd [options] [arg1] [arg2] ...") optparser = GentleOptionParser(usage=usage) @@ -930,7 +930,7 @@ def parse(cls, args): return v -class PigCommand(Command): +class PigCommand(Command): # skipcq PY-D0002 usage = ("pigcmd [options] [key1=value1] [key2=value2] ...") optparser = GentleOptionParser(usage=usage) @@ -1033,7 +1033,7 @@ def parse(cls, args): return v -class DbExportCommand(Command): +class DbExportCommand(Command): # skipcq PY-D0002 usage = ("dbexportcmd [options]") optparser = GentleOptionParser(usage=usage) @@ -1146,7 +1146,7 @@ def parse(cls, args): return v -class DbImportCommand(Command): +class DbImportCommand(Command): # skipcq PY-D0002 usage = "dbimportcmd [options]" optparser = GentleOptionParser(usage=usage) @@ -1230,7 +1230,7 @@ def parse(cls, args): return v -class CompositeCommand(Command): +class CompositeCommand(Command): # skipcq PY-D0002 @classmethod def compose(cls, sub_commands, macros=None, cluster_label=None, notify=False, name=None, tags=None): """ @@ -1259,7 +1259,7 @@ def compose(cls, sub_commands, macros=None, cluster_label=None, notify=False, na } -class DbTapQueryCommand(Command): +class DbTapQueryCommand(Command): # skipcq PY-D0002 usage = "dbtapquerycmd [options]" optparser = GentleOptionParser(usage=usage) @@ -1339,7 +1339,7 @@ def parse(cls, args): return v -class JupyterNotebookCommand(Command): +class JupyterNotebookCommand(Command): # skipcq PY-D0002 usage = "jupyternotebookcmd [options]" optparser = GentleOptionParser(usage=usage) From 7a924c20b7c9a25fc56c23acf605c9745e9aa01a Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:28:28 +0900 Subject: [PATCH 29/34] Fix FLK-D202: No blank lines allowed after function docstring --- qds_sdk/commands.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 0ad08d4d..41871b00 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -120,7 +120,6 @@ def create(cls, **kwargs): Returns: Command object """ - conn = Qubole.agent() if kwargs.get('command_type') is None: kwargs['command_type'] = cls.__name__ @@ -141,7 +140,6 @@ def run(cls, **kwargs): Returns: Command object """ - # vars to keep track of actual logs bytes (err, tmp) and new bytes seen in each iteration err_pointer, tmp_pointer, new_bytes = 0, 0, 0 print_logs_live = kwargs.pop("print_logs_live", None) # We don't want to send this to the API. @@ -380,7 +378,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.query is None and options.script_location is None: @@ -466,7 +463,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.query is None and options.script_location is None: @@ -716,7 +712,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.query is None and options.script_location is None: @@ -878,7 +873,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.inline is None and options.script_location is None: @@ -976,7 +970,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.latin_statements is None and options.script_location is None: @@ -1102,7 +1095,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.mode not in ["1", "2"]: @@ -1207,7 +1199,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.mode not in ["1", "2"]: @@ -1298,7 +1289,6 @@ def parse(cls, args): Raises: ParseError: when the arguments are not correct """ - try: (options, args) = cls.optparser.parse_args(args) if options.db_tap_id is None: @@ -1450,7 +1440,6 @@ def handler(self, signum, frame): # skipcq PY-D0003 def validate_json_input(string, option_type, cls): """Converts String to JSON and throws ParseError if string is not valid JSON""" - try: return json.loads(string) except ValueError as e: From 466a392237fd99d25b5bb46dd0851a0e58607aca Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:29:40 +0900 Subject: [PATCH 30/34] Fix FLK-D211: No blank lines allowed before class docstring --- qds_sdk/commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 41871b00..d405c520 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -29,7 +29,6 @@ class Command(Resource): - """ qds_sdk.Command is the base Qubole command class. Different types of Qubole commands can subclass this. From 5bf1ad4c9607cc167de9cbf9c622264722fb89b7 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:41:52 +0900 Subject: [PATCH 31/34] skipcq PYL-C0209 --- qds_sdk/commands.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index d405c520..7c4346f7 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -401,7 +401,7 @@ def parse(cls, args): try: q = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -486,7 +486,7 @@ def parse(cls, args): try: q = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -609,7 +609,7 @@ def validate_script_location(cls, options): # skipcq PY-D0003 elif fileExtension == ".sql": options.language = "sql" else: - raise ParseError("Invalid program type %s. Please choose one from python, scala, R or sql." % str(fileExtension), + raise ParseError("Invalid program type %s. Please choose one from python, scala, R or sql." % str(fileExtension), # skipcq PYL-C0209 cls.optparser.format_help()) if not _is_cloud_url(options.script_location): @@ -619,7 +619,7 @@ def validate_script_location(cls, options): # skipcq PY-D0003 try: q = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) @@ -734,7 +734,7 @@ def parse(cls, args): try: q = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -749,7 +749,7 @@ def parse(cls, args): class HadoopCommand(Command): # skipcq PY-D0002 subcmdlist = ["jar", "s3distcp", "streaming"] - usage = "hadoopcmd [options] <%s> [arg2] ..." % "|".join(subcmdlist) + usage = "hadoopcmd [options] <%s> [arg2] ..." % "|".join(subcmdlist) # skipcq PYL-C0209 optparser = GentleOptionParser(usage=usage) optparser.add_option("--cluster-label", dest="label", @@ -813,7 +813,7 @@ def parse(cls, args): subcmd = args.pop(0) if subcmd not in cls.subcmdlist: - raise ParseError("First argument must be one of <%s>" % + raise ParseError("First argument must be one of <%s>" % # skipcq PYL-C0209 "|".join(cls.subcmdlist)) parsed["sub_command"] = subcmd @@ -896,7 +896,7 @@ def parse(cls, args): try: s = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -993,7 +993,7 @@ def parse(cls, args): try: s = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -1310,7 +1310,7 @@ def parse(cls, args): try: q = open(options.script_location).read() except IOError as e: - raise ParseError("Unable to open script location: %s" % + raise ParseError("Unable to open script location: %s" % # skipcq PYL-C0209 str(e), cls.optparser.format_help()) options.script_location = None @@ -1442,7 +1442,7 @@ def validate_json_input(string, option_type, cls): try: return json.loads(string) except ValueError as e: - raise ParseError("Given %s is not valid JSON: %s" % (option_type, str(e)), + raise ParseError("Given %s is not valid JSON: %s" % (option_type, str(e)), # skipcq PYL-C0209 cls.optparser.format_help()) @@ -1504,7 +1504,7 @@ def _callback(downloaded, total): # skipcq PTC-W0065 if total in (0, downloaded): return progress = downloaded*100/total - sys.stderr.write('\r[{0}] {1}%'.format('#'*progress, progress)) + sys.stderr.write('\r[{0}] {1}%'.format('#'*progress, progress)) # skipcq PYL-C0209 sys.stderr.flush() m = _URI_RE.match(s3_path) From f670a54c3ede5d513d62a01876b8298e7346943a Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:46:20 +0900 Subject: [PATCH 32/34] Fix PYL-W1202: Lazy formatting of message string passed to logging module --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 7c4346f7..425d578e 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -149,7 +149,7 @@ def run(cls, **kwargs): while not Command.is_done(cmd.status): if sighandler.received_term_signal: - logging.warning("Received signal {}. Canceling Qubole Command ID: {}".format(sighandler.last_signal, cmd.id)) + logging.warning("Received signal %s. Canceling Qubole Command ID: %s" sighandler.last_signal, cmd.id) cls.cancel(cmd) sys.exit() time.sleep(Qubole.poll_interval) From aab8ba295f323e84534cfaf558772152f2a7ab20 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:47:17 +0900 Subject: [PATCH 33/34] skiqcq PY-R1000 --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 425d578e..3f1daa29 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -955,7 +955,7 @@ class PigCommand(Command): # skipcq PY-D0002 optparser.add_option("--retry", dest="retry", choices=[1, 2, 3], default=0, help="Number of retries for a job") @classmethod - def parse(cls, args): + def parse(cls, args): # skipcq PY-R1000 """ Parse command line arguments to construct a dictionary of command parameters that can be used to create a command From bbed02af80492d1a979a00522af2009e9628d6b9 Mon Sep 17 00:00:00 2001 From: winebarrel Date: Mon, 16 Oct 2023 00:48:31 +0900 Subject: [PATCH 34/34] Fix syntax error --- qds_sdk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qds_sdk/commands.py b/qds_sdk/commands.py index 3f1daa29..a0615524 100755 --- a/qds_sdk/commands.py +++ b/qds_sdk/commands.py @@ -149,7 +149,7 @@ def run(cls, **kwargs): while not Command.is_done(cmd.status): if sighandler.received_term_signal: - logging.warning("Received signal %s. Canceling Qubole Command ID: %s" sighandler.last_signal, cmd.id) + logging.warning("Received signal %s. Canceling Qubole Command ID: %s", sighandler.last_signal, cmd.id) cls.cancel(cmd) sys.exit() time.sleep(Qubole.poll_interval)