From 0219155ad932a4238ddced2f48bb6804ed8c3460 Mon Sep 17 00:00:00 2001 From: David Roe Date: Sat, 26 Aug 2023 00:09:29 -0400 Subject: [PATCH 01/21] Fix a bug in upload_from_file when extra columns present --- lmfdb/backend/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index e61a0132da..4c0d90ab42 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -1053,7 +1053,7 @@ def update_from_file( raise ValueError("You must specify a column that is contained in the datafile and uniquely specifies each row") with open(datafile) as F: tables = [self.search_table] - columns = self.search_cols + columns = list(self.search_cols) if self.extra_table is not None: tables.append(self.extra_table) columns.extend(self.extra_cols) From dec93cef0e52f084d673a955100ddc999622185f Mon Sep 17 00:00:00 2001 From: David Roe Date: Sat, 26 Aug 2023 10:28:55 -0400 Subject: [PATCH 02/21] Add missing Identifier --- lmfdb/backend/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index 4c0d90ab42..b4bee6b874 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -1141,7 +1141,7 @@ def drop_tmp(): ordered = False if etable is not None: ecols = SQL(", ").join([ - SQL("{0} = {1}.{0}").format(col, Identifier(tmp_table)) + SQL("{0} = {1}.{0}").format(Identifier(col), Identifier(tmp_table)) for col in ecols ]) self._execute(updater.format( From f8fa78068d25dd9edbf2ebf3098ef944199f91e9 Mon Sep 17 00:00:00 2001 From: David Roe Date: Tue, 29 Aug 2023 06:01:12 -0400 Subject: [PATCH 03/21] Add option for different tablespace to create_table --- lmfdb/backend/database.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index 66b24dcac7..42e546f91c 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -513,7 +513,7 @@ def _create_meta_tables_hist(self): print("Table meta_tables_hist created") - def create_table_like(self, new_name, table, data=False, commit=True): + def create_table_like(self, new_name, table, tablespace=None, data=False, commit=True): """ Copies the schema from an existing table, but none of the data, indexes or stats. @@ -521,6 +521,7 @@ def create_table_like(self, new_name, table, data=False, commit=True): - ``new_name`` -- a string giving the desired table name. - ``table`` -- a string or PostgresSearchTable object giving an existing table. + - ``tablespace`` -- the tablespace for the new table """ if isinstance(table, str): table = self[table] @@ -558,6 +559,7 @@ def create_table_like(self, new_name, table, data=False, commit=True): extra_columns, search_order, extra_order, + tablespace=tablespace, commit=commit, ) if data: @@ -591,6 +593,7 @@ def create_table( extra_columns=None, search_order=None, extra_order=None, + tablespace=None, force_description=False, commit=True, ): @@ -618,6 +621,7 @@ def create_table( in the search table, speeding up scans. - ``search_order`` -- (optional) list of column names, specifying the default order of columns - ``extra_order`` -- (optional) list of column names, specifying the default order of columns + - ``tablespace`` -- (optional) a postgres tablespace to use for the new table - ``force_description`` -- whether to require descriptions COMMON TYPES: @@ -733,6 +737,8 @@ def process_columns(coldict, colorder): creator = SQL("CREATE TABLE {0} ({1})").format( Identifier(name), SQL(", ").join(processed_search_columns) ) + if tablespace is not None: + creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name) if extra_columns is not None: @@ -741,6 +747,8 @@ def process_columns(coldict, colorder): Identifier(name + "_extras"), SQL(", ").join(processed_extra_columns), ) + if tablespace is not None: + creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name + "_extras") creator = SQL( @@ -758,6 +766,8 @@ def process_columns(coldict, colorder): "constraint_cols jsonb, constraint_values jsonb, threshold integer)" ) creator = creator.format(Identifier(name + "_stats")) + if tablespace is not None: + creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name + "_stats") self.grant_insert(name + "_stats") From d0af768ced2214375f559d3e454151ea8ff243e9 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 30 Aug 2023 14:42:17 -0400 Subject: [PATCH 04/21] Make indexes live in same tablespace as their search table, add option to copy over indexes in create_table_like --- lmfdb/backend/base.py | 22 ++++++++++++++++++---- lmfdb/backend/database.py | 30 ++++++++++++++++++------------ lmfdb/backend/statstable.py | 2 +- lmfdb/backend/table.py | 21 ++++++++++++++------- 4 files changed, 51 insertions(+), 24 deletions(-) diff --git a/lmfdb/backend/base.py b/lmfdb/backend/base.py index 9c04bdf443..c294efe180 100644 --- a/lmfdb/backend/base.py +++ b/lmfdb/backend/base.py @@ -212,7 +212,7 @@ class PostgresBase(): to the postgres database, as well as a name used when creating a logger. """ - def __init__(self, loggername, db): + def __init__(self, loggername, db, tablespace=None): # Have to record this object in the db so that we can reset the connection if necessary. # This function also sets self.conn db.register_object(self) @@ -232,6 +232,7 @@ def __init__(self, loggername, db): shandler = logging.StreamHandler() shandler.setFormatter(formatter) l.addHandler(shandler) + self.tablespace = tablespace def _execute( self, @@ -902,6 +903,17 @@ def _copy_from(self, filename, table, columns, header, kwds): return addid, cur.rowcount + def _tablespace_clause(self, tablespace=None): + """ + A clause for use in CREATE statements + """ + if tablespace is None: + tablespace = self.tablespace + if tablespace is None: + return SQL("") + else: + return SQL(" TABLESPACE {0}").format(Identifier(tablespace)) + def _clone(self, table, tmp_table): """ Utility function: creates a table with the same schema as the given one. @@ -921,7 +933,7 @@ def _clone(self, table, tmp_table): "Run db.%s.cleanup_from_reload() if you want to delete it and proceed." % (tmp_table, table) ) - creator = SQL("CREATE TABLE {0} (LIKE {1})").format(Identifier(tmp_table), Identifier(table)) + creator = SQL("CREATE TABLE {0} (LIKE {1}){2}").format(Identifier(tmp_table), Identifier(table), self._tablespace_clause()) self._execute(creator) def _check_col_datatype(self, typ): @@ -931,7 +943,9 @@ def _check_col_datatype(self, typ): def _create_table(self, name, columns): """ - Utility function: creates a table with the schema specified by ``columns`` + Utility function: creates a table with the schema specified by ``columns``. + + If self is a table, the new table will be in the same tablespace. INPUT: @@ -943,7 +957,7 @@ def _create_table(self, name, columns): for col, typ in columns: self._check_col_datatype(typ) table_col = SQL(", ").join(SQL("{0} %s" % typ).format(Identifier(col)) for col, typ in columns) - creator = SQL("CREATE TABLE {0} ({1})").format(Identifier(name), table_col) + creator = SQL("CREATE TABLE {0} ({1}){2}").format(Identifier(name), table_col, self._tablespace_clause()) self._execute(creator) def _create_table_from_header(self, filename, name, sep, addid=True): diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index 42e546f91c..c581dc5f24 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -513,7 +513,7 @@ def _create_meta_tables_hist(self): print("Table meta_tables_hist created") - def create_table_like(self, new_name, table, tablespace=None, data=False, commit=True): + def create_table_like(self, new_name, table, tablespace=None, data=False, indexes=False, commit=True): """ Copies the schema from an existing table, but none of the data, indexes or stats. @@ -522,6 +522,8 @@ def create_table_like(self, new_name, table, tablespace=None, data=False, commit - ``new_name`` -- a string giving the desired table name. - ``table`` -- a string or PostgresSearchTable object giving an existing table. - ``tablespace`` -- the tablespace for the new table + - ``data`` -- whether to copy over data from the source table + - ``indexes`` -- whether to copy over indexes from the source table """ if isinstance(table, str): table = self[table] @@ -579,6 +581,10 @@ def create_table_like(self, new_name, table, tablespace=None, data=False, commit ), commit=commit, ) + if indexes: + for idata in table.list_indexes(verbose=False).values(): + self[new_name].create_index(**idata) + if data: self[new_name].stats.refresh_stats() def create_table( @@ -733,39 +739,39 @@ def process_columns(coldict, colorder): if col_description is None: col_description = {col: "" for col in description_columns} + tablespace = self._tablespace_clause(tablespace) with DelayCommit(self, commit, silence=True): - creator = SQL("CREATE TABLE {0} ({1})").format( - Identifier(name), SQL(", ").join(processed_search_columns) + creator = SQL("CREATE TABLE {0} ({1}){2}").format( + Identifier(name), + SQL(", ").join(processed_search_columns), + tablespace, ) - if tablespace is not None: - creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name) if extra_columns is not None: - creator = SQL("CREATE TABLE {0} ({1})") + creator = SQL("CREATE TABLE {0} ({1}){2}") creator = creator.format( Identifier(name + "_extras"), SQL(", ").join(processed_extra_columns), + tablespace, ) - if tablespace is not None: - creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name + "_extras") creator = SQL( "CREATE TABLE {0} " "(cols jsonb, values jsonb, count bigint, " - "extra boolean, split boolean DEFAULT FALSE)" + "extra boolean, split boolean DEFAULT FALSE){1}" ) - creator = creator.format(Identifier(name + "_counts")) + creator = creator.format(Identifier(name + "_counts"), tablespace) self._execute(creator) self.grant_select(name + "_counts") self.grant_insert(name + "_counts") creator = SQL( "CREATE TABLE {0} " '(cols jsonb, stat text COLLATE "C", value numeric, ' - "constraint_cols jsonb, constraint_values jsonb, threshold integer)" + "constraint_cols jsonb, constraint_values jsonb, threshold integer){1}" ) - creator = creator.format(Identifier(name + "_stats")) + creator = creator.format(Identifier(name + "_stats"), tablespace) if tablespace is not None: creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) diff --git a/lmfdb/backend/statstable.py b/lmfdb/backend/statstable.py index b5406781a7..b3908b6f69 100644 --- a/lmfdb/backend/statstable.py +++ b/lmfdb/backend/statstable.py @@ -205,7 +205,7 @@ class PostgresStatsTable(PostgresBase): saving = False def __init__(self, table, total=None): - PostgresBase.__init__(self, table.search_table, table._db) + PostgresBase.__init__(self, table.search_table, table._db, table._get_tablespace()) self.table = table self.search_table = st = table.search_table self.stats = st + "_stats" diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index e61a0132da..684a3aebb4 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -148,7 +148,7 @@ def __init__( self._out_of_order = out_of_order self._stats_valid = stats_valid self._include_nones = include_nones - PostgresBase.__init__(self, search_table, db) + PostgresBase.__init__(self, search_table, db, self._get_tablespace()) self.col_type = {} self.has_id = False self.search_cols = [] @@ -275,8 +275,14 @@ def list_indexes(self, verbose=False): if not verbose: return output - @staticmethod - def _create_index_statement(name, table, type, columns, modifiers, storage_params): + def _get_tablespace(self): + """ + Determine the tablespace hosting this table (which is then used for indexes and constraints) + """ + cur = self._execute(SQL("SELECT tablespace FROM pg_tables WHERE tablename=%s"), [self.search_table]) + return cur.fetchone()[0] + + def _create_index_statement(self, name, table, type, columns, modifiers, storage_params): """ Utility function for making the create index SQL statement. """ @@ -290,6 +296,7 @@ def _create_index_statement(name, table, type, columns, modifiers, storage_param ) else: storage_params = SQL("") + tablespace = self._tablespace_clause() modifiers = [" " + " ".join(mods) if mods else "" for mods in modifiers] # The inner % operator is on strings prior to being wrapped by SQL: modifiers have been whitelisted. columns = SQL(", ").join( @@ -297,8 +304,8 @@ def _create_index_statement(name, table, type, columns, modifiers, storage_param for col, mods in zip(columns, modifiers) ) # The inner % operator is on strings prior to being wrapped by SQL: type has been whitelisted. - creator = SQL("CREATE INDEX {0} ON {1} USING %s ({2}){3}" % (type)) - return creator.format(Identifier(name), Identifier(table), columns, storage_params) + creator = SQL("CREATE INDEX {0} ON {1} USING %s ({2}){3}{4}" % (type)) + return creator.format(Identifier(name), Identifier(table), columns, storage_params, tablespace) def _create_counts_indexes(self, suffix="", warning_only=False): """ @@ -1079,7 +1086,7 @@ def drop_tmp(): SQL("{0} " + self.col_type[col]).format(Identifier(col)) for col in columns ]) - creator = SQL("CREATE TABLE {0} ({1})").format(Identifier(tmp_table), processed_columns) + creator = SQL("CREATE TABLE {0} ({1}){2}").format(Identifier(tmp_table), processed_columns, self._tablespace_clause()) self._execute(creator) # We need to add an id column and populate it correctly if label_col != "id": @@ -2438,7 +2445,7 @@ def create_extra_table(self, columns, ordered=False, sep="|", commit=True): col_type_SQL = SQL(", ").join( SQL("{0} %s" % typ).format(Identifier(col)) for col, typ in col_type ) - creator = SQL("CREATE TABLE {0} ({1})").format(Identifier(self.extra_table), col_type_SQL) + creator = SQL("CREATE TABLE {0} ({1}){2}").format(Identifier(self.extra_table), col_type_SQL, self._tablespace_clause()) self._execute(creator) if columns: self.drop_constraints(columns) From 6a9b7d790f4b3ac13c50929f1a218db837439885 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 30 Aug 2023 14:56:06 -0400 Subject: [PATCH 05/21] Delay _get_tablespace until it's needed --- lmfdb/backend/base.py | 6 +++++- lmfdb/backend/statstable.py | 6 +++++- lmfdb/backend/table.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lmfdb/backend/base.py b/lmfdb/backend/base.py index c294efe180..cdef2c4bc6 100644 --- a/lmfdb/backend/base.py +++ b/lmfdb/backend/base.py @@ -903,12 +903,16 @@ def _copy_from(self, filename, table, columns, header, kwds): return addid, cur.rowcount + def _get_tablespace(self): + # overridden in table and statstable + pass + def _tablespace_clause(self, tablespace=None): """ A clause for use in CREATE statements """ if tablespace is None: - tablespace = self.tablespace + tablespace = self._get_tablespace() if tablespace is None: return SQL("") else: diff --git a/lmfdb/backend/statstable.py b/lmfdb/backend/statstable.py index b3908b6f69..1fca522599 100644 --- a/lmfdb/backend/statstable.py +++ b/lmfdb/backend/statstable.py @@ -205,7 +205,7 @@ class PostgresStatsTable(PostgresBase): saving = False def __init__(self, table, total=None): - PostgresBase.__init__(self, table.search_table, table._db, table._get_tablespace()) + PostgresBase.__init__(self, table.search_table, table._db) self.table = table self.search_table = st = table.search_table self.stats = st + "_stats" @@ -216,6 +216,10 @@ def __init__(self, table, total=None): total = self._slow_count({}, extra=False) self.total = total + def _get_tablespace(self): + # We use the same tablespace for stats and counts tables as for the main search table + return self.table._get_tablespace() + def _has_stats(self, jcols, ccols, cvals, threshold, split_list=False, threshold_inequality=False, suffix=""): """ Checks whether statistics have been recorded for a given set of columns. diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index 684a3aebb4..83a31197f4 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -148,7 +148,7 @@ def __init__( self._out_of_order = out_of_order self._stats_valid = stats_valid self._include_nones = include_nones - PostgresBase.__init__(self, search_table, db, self._get_tablespace()) + PostgresBase.__init__(self, search_table, db) self.col_type = {} self.has_id = False self.search_cols = [] From 95e66b7823915aef350212baeb443e9767c277b9 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 30 Aug 2023 15:23:13 -0400 Subject: [PATCH 06/21] Add array_ops as a valid gin modifier --- lmfdb/backend/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index 83a31197f4..3669916bbd 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -29,7 +29,7 @@ "varchar_ops", "varchar_pattern_ops", ], - "gin": ["jsonb_path_ops"], + "gin": ["jsonb_path_ops", "array_ops"], "gist": ["inet_ops"], "hash": [ "bpchar_pattern_ops", From 987c8a9b9317e8339b6fc07e304de544672098d1 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 30 Aug 2023 16:08:46 -0400 Subject: [PATCH 07/21] Remove bad clause --- lmfdb/backend/database.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index c581dc5f24..816745ce61 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -772,8 +772,6 @@ def process_columns(coldict, colorder): "constraint_cols jsonb, constraint_values jsonb, threshold integer){1}" ) creator = creator.format(Identifier(name + "_stats"), tablespace) - if tablespace is not None: - creator += SQL(" TABLESPACE {0}").format(Identifier(tablespace)) self._execute(creator) self.grant_select(name + "_stats") self.grant_insert(name + "_stats") From 1dde33a991537f65ef92e88e257c4cf9a7092e4b Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 30 Aug 2023 17:16:47 -0400 Subject: [PATCH 08/21] Remove tablespace from PostgresBase.__init__ --- lmfdb/backend/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lmfdb/backend/base.py b/lmfdb/backend/base.py index cdef2c4bc6..2016181df7 100644 --- a/lmfdb/backend/base.py +++ b/lmfdb/backend/base.py @@ -212,7 +212,7 @@ class PostgresBase(): to the postgres database, as well as a name used when creating a logger. """ - def __init__(self, loggername, db, tablespace=None): + def __init__(self, loggername, db): # Have to record this object in the db so that we can reset the connection if necessary. # This function also sets self.conn db.register_object(self) @@ -232,7 +232,6 @@ def __init__(self, loggername, db, tablespace=None): shandler = logging.StreamHandler() shandler.setFormatter(formatter) l.addHandler(shandler) - self.tablespace = tablespace def _execute( self, From 36accc55ce78b3bbb82935370bfe13fef684b029 Mon Sep 17 00:00:00 2001 From: David Roe Date: Thu, 31 Aug 2023 04:36:17 -0400 Subject: [PATCH 09/21] Choose index names to avoid collisions --- lmfdb/backend/table.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lmfdb/backend/table.py b/lmfdb/backend/table.py index 3669916bbd..65cc88b4f5 100644 --- a/lmfdb/backend/table.py +++ b/lmfdb/backend/table.py @@ -453,6 +453,13 @@ def mod(col): name = "_".join([self.search_table] + [col[:2] for col in columns]) else: name = "_".join([self.search_table] + ["".join(col[0] for col in columns)]) + if len(name) >= 64: + name = name[:63] + if self._relation_exists(name): + disamb = 0 + while self._relation_exists(name + str(disamb)): + disamb += 1 + name += str(disamb) with DelayCommit(self, silence=True): self._check_index_name(name, "Index") From 9eb5852a4d64a62a295378121b066881ce2585d2 Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 1 Sep 2023 03:42:50 -0400 Subject: [PATCH 10/21] Add tablespaces to api/stats --- lmfdb/api/api.py | 5 ++++- lmfdb/api/templates/api-stats.html | 2 ++ lmfdb/backend/database.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py index e2eaad7d57..d46da04036 100644 --- a/lmfdb/api/api.py +++ b/lmfdb/api/api.py @@ -96,6 +96,7 @@ def split_db(tablename): dname, _ = split_db(tablename) dbSize[dname] += sizes['total_bytes'] dbObjects[dname] += sizes['nrows'] + tablespaces = db.tablespaces() for tablename, sizes in table_sizes.items(): tsize = sizes['total_bytes'] size += tsize @@ -119,7 +120,9 @@ def split_db(tablename): 'size': csize, 'avgObjSize':avg_size, 'indexSize':mb(sizes['index_bytes']), 'dataSize':mb(sizes['table_bytes'] + sizes['toast_bytes'] + sizes['extra_bytes']), 'countsSize':mb(sizes['counts_bytes']), 'statsSize':mb(sizes['stats_bytes']), - 'nrows': sizes['nrows'], 'nstats': sizes['nstats'], 'ncounts': sizes['ncounts']} + 'nrows': sizes['nrows'], 'nstats': sizes['nstats'], 'ncounts': sizes['ncounts'], + 'tablespace': tablespaces[tablename], + } dataSize = size - indexSize info['ntables'] = len(table_sizes) info['nobjects'] = nobjects diff --git a/lmfdb/api/templates/api-stats.html b/lmfdb/api/templates/api-stats.html index dc96b18a49..2964b72ade 100644 --- a/lmfdb/api/templates/api-stats.html +++ b/lmfdb/api/templates/api-stats.html @@ -52,6 +52,7 @@ Index (MiB) Objects Average Size (B) + Tablespace @@ -64,6 +65,7 @@ {{x.indexSize}} {{x.nrows}} {{x.avgObjSize}} + {{x.tablespace}} {% endfor %} diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index 816745ce61..2e757e97e1 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -1284,3 +1284,10 @@ def show_locks(self): ) else: print("No locks currently held") + + def tablespaces(self): + """ + Returns a dictionary giving giving the tablespace for all tables + """ + D = {rec[0]: rec[1] for rec in self._execute(SQL("SELECT tablename, tablespace FROM pg_tables"))} + return {name: space if space else "" for (name, space) in D.items() if name in self.tablenames} From c967cfeff381dfea34f84510fc5874437982047b Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 1 Sep 2023 03:46:11 -0400 Subject: [PATCH 11/21] Include all tables in tablespaces() output --- lmfdb/backend/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/backend/database.py b/lmfdb/backend/database.py index 2e757e97e1..d3f1ff2846 100644 --- a/lmfdb/backend/database.py +++ b/lmfdb/backend/database.py @@ -1290,4 +1290,4 @@ def tablespaces(self): Returns a dictionary giving giving the tablespace for all tables """ D = {rec[0]: rec[1] for rec in self._execute(SQL("SELECT tablename, tablespace FROM pg_tables"))} - return {name: space if space else "" for (name, space) in D.items() if name in self.tablenames} + return {name: space if space else "" for (name, space) in D.items()} From d965d880774475e9df2c70121f21a89cc231dd53 Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 29 Sep 2023 14:01:17 -0400 Subject: [PATCH 12/21] Fix broken test --- lmfdb/groups/abstract/test_abstract_groups.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lmfdb/groups/abstract/test_abstract_groups.py b/lmfdb/groups/abstract/test_abstract_groups.py index 55a05e0307..f192442082 100644 --- a/lmfdb/groups/abstract/test_abstract_groups.py +++ b/lmfdb/groups/abstract/test_abstract_groups.py @@ -92,5 +92,4 @@ def test_underlying_data(self): self.check_args("/Groups/Abstract/sdata/16.8.2.b1.a1", [ "gps_subgroups", "16.8.2.b1.a1", "gps_groups", "[28776, 16577, 5167]", # perm_gens - "[16582, 136, 5167, 40176]", # perm_gens "[[1, 1, 1]]"]) # faithful_reps From 05e9602c6c117add379ba3f46e1b576c47e3567f Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 29 Sep 2023 14:05:27 -0400 Subject: [PATCH 13/21] Fix broken test --- lmfdb/groups/abstract/test_abstract_groups.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lmfdb/groups/abstract/test_abstract_groups.py b/lmfdb/groups/abstract/test_abstract_groups.py index 55a05e0307..f192442082 100644 --- a/lmfdb/groups/abstract/test_abstract_groups.py +++ b/lmfdb/groups/abstract/test_abstract_groups.py @@ -92,5 +92,4 @@ def test_underlying_data(self): self.check_args("/Groups/Abstract/sdata/16.8.2.b1.a1", [ "gps_subgroups", "16.8.2.b1.a1", "gps_groups", "[28776, 16577, 5167]", # perm_gens - "[16582, 136, 5167, 40176]", # perm_gens "[[1, 1, 1]]"]) # faithful_reps From 332036617de01a4b2959b2dbdc5794397d76c9b6 Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 29 Sep 2023 14:29:51 -0400 Subject: [PATCH 14/21] Try to fix sorting bug --- lmfdb/api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py index efab36d3a2..fbe82062c1 100644 --- a/lmfdb/api/api.py +++ b/lmfdb/api/api.py @@ -121,7 +121,7 @@ def split_db(tablename): 'indexSize':mb(sizes['index_bytes']), 'dataSize':mb(sizes['table_bytes'] + sizes['toast_bytes'] + sizes['extras_bytes']), 'countsSize':mb(sizes['counts_bytes']), 'statsSize':mb(sizes['stats_bytes']), 'nrows': sizes['nrows'], 'nstats': sizes['nstats'], 'ncounts': sizes['ncounts'], - 'tablespace': tablespaces[tablename], + 'tablespace': tablespaces.get(tablename, ""), } dataSize = size - indexSize info['ntables'] = len(table_sizes) From 76c02d43a77c21d8a4a7a728df180fdebfae47bd Mon Sep 17 00:00:00 2001 From: Barinder S Banwait Date: Fri, 29 Sep 2023 14:48:32 -0400 Subject: [PATCH 15/21] fix bug --- lmfdb/characters/ListCharacters.py | 2 +- lmfdb/characters/templates/ModulusList.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lmfdb/characters/ListCharacters.py b/lmfdb/characters/ListCharacters.py index 0b5200a2db..d604e71562 100644 --- a/lmfdb/characters/ListCharacters.py +++ b/lmfdb/characters/ListCharacters.py @@ -71,7 +71,7 @@ def get_character_modulus(a, b, limit=7): entry.append(el) entries[(row, col)] = entry entries2 = {} - def out(chi): return (chi.number, chi.is_primitive, + def out(chi): return (chi.number, chi.is_primitive(), chi.order, chi.is_even()) for k, v in entries.items(): l = [] diff --git a/lmfdb/characters/templates/ModulusList.html b/lmfdb/characters/templates/ModulusList.html index f88a003e22..13cf1ebf89 100644 --- a/lmfdb/characters/templates/ModulusList.html +++ b/lmfdb/characters/templates/ModulusList.html @@ -105,7 +105,7 @@ {% for e in entry %} -
+ {% for chi in e %} ${{ chi[0] }}$ {% if loop.first %}
{% endif %} From b3301f2e8722d840de4ee654d76f9fbfb0abcca5 Mon Sep 17 00:00:00 2001 From: AndrewVSutherland Date: Fri, 29 Sep 2023 14:51:42 -0400 Subject: [PATCH 16/21] fix conrey knowl typo --- lmfdb/characters/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/characters/main.py b/lmfdb/characters/main.py index a1347d4b0e..b2ad3fbf02 100644 --- a/lmfdb/characters/main.py +++ b/lmfdb/characters/main.py @@ -233,7 +233,7 @@ def display_galois_orbit(modulus, first_label, last_label, degree): character_columns = SearchColumns([ LinkCol("label", "character.dirichlet.galois_orbit_label", "Orbit label", lambda label: label.replace(".", "/"), default=True, align="center"), - MultiProcessedCol("conrey", "character.dirichlet.conrey'", "Conrey labels", ["modulus", "first_label", "last_label", "degree"], + MultiProcessedCol("conrey", "character.dirichlet.conrey", "Conrey labels", ["modulus", "first_label", "last_label", "degree"], display_galois_orbit, default=True, align="center", short_title="Conrey labels"), MathCol("modulus", "character.dirichlet.modulus", "Modulus", default=True), MathCol("conductor", "character.dirichlet.conductor", "Conductor", default=True), From f118808fe77cee9a2dbfdccb4bf50c81428cbf6a Mon Sep 17 00:00:00 2001 From: David Roe Date: Fri, 29 Sep 2023 15:00:32 -0400 Subject: [PATCH 17/21] Make g2c empty find work --- lmfdb/genus2_curves/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/genus2_curves/main.py b/lmfdb/genus2_curves/main.py index d644d042fc..34f2787eb0 100644 --- a/lmfdb/genus2_curves/main.py +++ b/lmfdb/genus2_curves/main.py @@ -194,7 +194,7 @@ def index(): @g2c_page.route("/Q/") def index_Q(): info = to_dict(request.args, search_array=G2CSearchArray()) - if len(info) > 1: + if request.args: return genus2_curve_search(info) info["stats"] = G2C_stats() info["stats_url"] = url_for(".statistics") From 0edd4ab55cdc45d5fcfca300b4e1f9e334b74758 Mon Sep 17 00:00:00 2001 From: Edgar Costa Date: Fri, 29 Sep 2023 17:22:45 -0400 Subject: [PATCH 18/21] fix origins for Dirichlet L-fcns --- lmfdb/utils/names_and_urls.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lmfdb/utils/names_and_urls.py b/lmfdb/utils/names_and_urls.py index b8ad0ec834..e8d5d0340c 100644 --- a/lmfdb/utils/names_and_urls.py +++ b/lmfdb/utils/names_and_urls.py @@ -125,10 +125,14 @@ def name_and_object_from_url(url, check_existence=False): else: name = 'Sato Tate group $%s$' % name obj_exists = True + elif url_split[:2] == ["Character", "Dirichlet"]: + modulus = int(url_split[2]) + conrey = int(url_split[3]) + name = "Character $\chi_{%d}(%d, \cdot)$" % (modulus, conrey) + obj_exists = True else: # FIXME - #print("unknown url", url) - pass + assert False, url return name, obj_exists From b58e344f7a78731f499493de650394ac42b8ade8 Mon Sep 17 00:00:00 2001 From: edgarcosta Date: Fri, 29 Sep 2023 21:44:20 +0000 Subject: [PATCH 19/21] autopep8 action fixes --- lmfdb/utils/names_and_urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmfdb/utils/names_and_urls.py b/lmfdb/utils/names_and_urls.py index e8d5d0340c..36a233a38f 100644 --- a/lmfdb/utils/names_and_urls.py +++ b/lmfdb/utils/names_and_urls.py @@ -128,7 +128,7 @@ def name_and_object_from_url(url, check_existence=False): elif url_split[:2] == ["Character", "Dirichlet"]: modulus = int(url_split[2]) conrey = int(url_split[3]) - name = "Character $\chi_{%d}(%d, \cdot)$" % (modulus, conrey) + name = r"Character $\chi_{%d}(%d, \cdot)$" % (modulus, conrey) obj_exists = True else: # FIXME From e5159e124dbf205b01c1a20f0e61ce89c3230f29 Mon Sep 17 00:00:00 2001 From: AndrewVSutherland Date: Sat, 30 Sep 2023 14:31:25 -0400 Subject: [PATCH 20/21] fix bug in WebNewform.order_gen() for newforms over Q(i) (i is not necessarily the generator of the Hecke ring) --- lmfdb/classical_modular_forms/web_newform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lmfdb/classical_modular_forms/web_newform.py b/lmfdb/classical_modular_forms/web_newform.py index f253606344..88697a1e3f 100644 --- a/lmfdb/classical_modular_forms/web_newform.py +++ b/lmfdb/classical_modular_forms/web_newform.py @@ -814,9 +814,9 @@ def order_basis(self): return html % ("", self._order_basis_forward(), self._nu_latex, " nodisplay", self._order_basis_inverse(), self._nu_latex) def order_gen(self): - if self.field_poly_root_of_unity == 4: - return r'\(i = \sqrt{-1}\)' - elif (self.hecke_ring_power_basis or self.qexp_converted) and self.field_poly_is_cyclotomic: + if (self.hecke_ring_power_basis or self.qexp_converted) and self.field_poly_is_cyclotomic: + if self.field_poly_root_of_unity == 4: + return r'\(i = \sqrt{-1}\)' return r'a primitive root of unity \(\zeta_{%s}\)' % self.field_poly_root_of_unity elif self.dim == 2: c, b, a = map(ZZ, self.field_poly) From d67f856a9a6ee511a85b52475c17c63acb36f619 Mon Sep 17 00:00:00 2001 From: AndrewVSutherland Date: Sat, 30 Sep 2023 14:52:14 -0400 Subject: [PATCH 21/21] fix qexp display bug over Q(i) --- lmfdb/classical_modular_forms/web_newform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lmfdb/classical_modular_forms/web_newform.py b/lmfdb/classical_modular_forms/web_newform.py index 88697a1e3f..03e8fedc91 100644 --- a/lmfdb/classical_modular_forms/web_newform.py +++ b/lmfdb/classical_modular_forms/web_newform.py @@ -872,10 +872,10 @@ def _PrintRing(self): # univariate polynomial rings don't support order, # we work around it by introducing a dummy variable """ - if self.field_poly_root_of_unity == 4: - return PolynomialRing(QQ, 'i') m = self.hecke_ring_cyclotomic_generator if m is not None and m != 0: + if m == 4: + return PolynomialRing(QQ, 'i') return PolynomialRing(QQ, [self._zeta_print, 'dummy'], order='negdeglex') if self.single_generator: if (self.hecke_ring_power_basis or self.qexp_converted) and self.field_poly_is_cyclotomic: