From 30685ab79de08eca4f76311db52acecc5d088eaf Mon Sep 17 00:00:00 2001 From: flipmcf Date: Thu, 8 Jun 2023 11:45:35 -0400 Subject: [PATCH 1/6] logging error counts and writing error paths to file --- src/collective/exportimport/export_content.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index de398efc..d39c80e0 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -112,6 +112,7 @@ def __call__( download_to_server=False, migration=True, include_revisions=False, + write_errors=False ): self.portal_type = portal_type or [] if isinstance(self.portal_type, str): @@ -175,6 +176,9 @@ def __call__( filename = self.path.split("/")[-1] filename = "{}.json".format(filename) + self.write_errors = write_errors or self.request.form.get("write_errors") + + self.errors = [] content_generator = self.export_content() number = 0 @@ -197,9 +201,12 @@ def __call__( f.write(",") json.dump(datum, f, sort_keys=True, indent=4) if number: + if self.errors and self.write_errors: + errors = {"unexported_paths": self.errors} + json.dump(errors, f, indent=4) f.write("]") - msg = _(u"Exported {} items ({}) as {} to {}").format( - number, ", ".join(self.portal_type), filename, filepath + msg = _(u"Exported {} items ({}) as {} to {} with {} errors").format( + number, ", ".join(self.portal_type), filename, filepath, len(self.errors) ) logger.info(msg) api.portal.show_message(msg, self.request) @@ -221,8 +228,11 @@ def __call__( f.write(",") json.dump(datum, f, sort_keys=True, indent=4) if number: + if self.errors and self.write_errors: + errors = {"unexported_paths": self.errors} + json.dump(errors, f, indent=4) f.write("]") - msg = _(u"Exported {} {}").format(number, self.portal_type) + msg = _(u"Exported {} {} with {} errors").format(number, self.portal_type, len(self.errors)) logger.info(msg) api.portal.show_message(msg, self.request) response = self.request.response @@ -292,7 +302,9 @@ def export_content(self): try: obj = brain.getObject() except Exception: - logger.exception(u"Error getting brain %s", brain.getPath(), exc_info=True) + msg = f"Error getting brain {brain.getPath()}" + self.errors.append((None, msg)) + logger.exception(msg, exc_info=True) continue if obj is None: logger.error(u"brain.getObject() is None %s", brain.getPath()) @@ -313,7 +325,9 @@ def export_content(self): yield item except Exception: - logger.exception(u"Error exporting %s", obj.absolute_url(), exc_info=True) + msg = f"Error exporting {obj.absolute_url()}" + self.errors.append((obj.absolute_url(), msg)) + logger.exception(msg, exc_info=True) def portal_types(self): """A list with info on all content types with existing items.""" From cf4f670d4a7522849a4282cc682fd6d6c18e49ba Mon Sep 17 00:00:00 2001 From: flipmcf Date: Thu, 8 Jun 2023 13:30:00 -0400 Subject: [PATCH 2/6] wire up form --- src/collective/exportimport/export_content.py | 3 +-- .../exportimport/templates/export_content.pt | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index d39c80e0..9f983d5c 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -142,6 +142,7 @@ def __call__( ("2", _(u"as blob paths")), ) self.include_revisions = include_revisions + self.write_errors = write_errors or self.request.form.get("write_errors") self.update() @@ -176,8 +177,6 @@ def __call__( filename = self.path.split("/")[-1] filename = "{}.json".format(filename) - self.write_errors = write_errors or self.request.form.get("write_errors") - self.errors = [] content_generator = self.export_content() diff --git a/src/collective/exportimport/templates/export_content.pt b/src/collective/exportimport/templates/export_content.pt index 736a949d..a7672239 100644 --- a/src/collective/exportimport/templates/export_content.pt +++ b/src/collective/exportimport/templates/export_content.pt @@ -133,6 +133,22 @@ +
+ +
+
From ae1145bacbbc53e64515558b5c253d98aaab13ab Mon Sep 17 00:00:00 2001 From: flipmcf Date: Thu, 8 Jun 2023 13:50:19 -0400 Subject: [PATCH 3/6] change error output to dict of path & message --- src/collective/exportimport/export_content.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index 9f983d5c..c95caff7 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -302,11 +302,13 @@ def export_content(self): obj = brain.getObject() except Exception: msg = f"Error getting brain {brain.getPath()}" - self.errors.append((None, msg)) + self.errors.append({'path':None, 'message': msg}) logger.exception(msg, exc_info=True) continue if obj is None: - logger.error(u"brain.getObject() is None %s", brain.getPath()) + msg = f'brain.getObject() is None {brain.getPath()}' + logger.error(msg) + self.errors.append({'path':None, 'message': msg}) continue obj = self.global_obj_hook(obj) if not obj: @@ -325,7 +327,7 @@ def export_content(self): yield item except Exception: msg = f"Error exporting {obj.absolute_url()}" - self.errors.append((obj.absolute_url(), msg)) + self.errors.append({'path':obj.absolute_url(), 'message':msg}) logger.exception(msg, exc_info=True) def portal_types(self): From 40c2da54ee6144700fae2abf1c1c66c7565bcae8 Mon Sep 17 00:00:00 2001 From: flipmcf Date: Thu, 8 Jun 2023 13:57:17 -0400 Subject: [PATCH 4/6] write one more comma... --- src/collective/exportimport/export_content.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index c95caff7..f209f0c7 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -201,6 +201,7 @@ def __call__( json.dump(datum, f, sort_keys=True, indent=4) if number: if self.errors and self.write_errors: + f.write(",") errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") @@ -228,6 +229,7 @@ def __call__( json.dump(datum, f, sort_keys=True, indent=4) if number: if self.errors and self.write_errors: + f.write(",") errors = {"unexported_paths": self.errors} json.dump(errors, f, indent=4) f.write("]") From 4a6c4d70222ed195f1c409e2042718a55c6b7861 Mon Sep 17 00:00:00 2001 From: flipmcf Date: Thu, 8 Jun 2023 14:34:24 -0400 Subject: [PATCH 5/6] fix failing py27 tests --- src/collective/exportimport/export_content.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index f209f0c7..b0d6f4ca 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -303,12 +303,12 @@ def export_content(self): try: obj = brain.getObject() except Exception: - msg = f"Error getting brain {brain.getPath()}" + msg = u"Error getting brain {}".format(brain.getPath()) self.errors.append({'path':None, 'message': msg}) logger.exception(msg, exc_info=True) continue if obj is None: - msg = f'brain.getObject() is None {brain.getPath()}' + msg = u"brain.getObject() is None {}}".format(brain.getPath()) logger.error(msg) self.errors.append({'path':None, 'message': msg}) continue @@ -328,7 +328,7 @@ def export_content(self): yield item except Exception: - msg = f"Error exporting {obj.absolute_url()}" + msg = u"Error exporting {}}".format(obj.absolute_url()) self.errors.append({'path':obj.absolute_url(), 'message':msg}) logger.exception(msg, exc_info=True) From 108282834913ee4d125e326e43606775f93dfa89 Mon Sep 17 00:00:00 2001 From: flipmcf Date: Fri, 9 Jun 2023 16:13:20 -0400 Subject: [PATCH 6/6] remove rogue '}' in string format. --- src/collective/exportimport/export_content.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/collective/exportimport/export_content.py b/src/collective/exportimport/export_content.py index b0d6f4ca..060c1c05 100644 --- a/src/collective/exportimport/export_content.py +++ b/src/collective/exportimport/export_content.py @@ -308,7 +308,7 @@ def export_content(self): logger.exception(msg, exc_info=True) continue if obj is None: - msg = u"brain.getObject() is None {}}".format(brain.getPath()) + msg = u"brain.getObject() is None {}".format(brain.getPath()) logger.error(msg) self.errors.append({'path':None, 'message': msg}) continue @@ -328,7 +328,7 @@ def export_content(self): yield item except Exception: - msg = u"Error exporting {}}".format(obj.absolute_url()) + msg = u"Error exporting {}".format(obj.absolute_url()) self.errors.append({'path':obj.absolute_url(), 'message':msg}) logger.exception(msg, exc_info=True)