forked from Arelle/EdgarRenderer
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path__init__.py
2084 lines (1926 loc) · 133 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-
"""
:mod:`EdgarRenderer.EdgarRenderer`
~~~~~~~~~~~~~~~~~~~
Edgar(tm) Renderer was created by staff of the U.S. Securities and Exchange Commission.
Data and content created by government employees within the scope of their employment
are not subject to domestic copyright protection. 17 U.S.C. 105.
This plug-in is an adaptation of the gitHub arelle/EdgarRenderer EdgarRenderer.py into
a plug-in of the Arelle environment, including RESTful web use. It replaces main
program and daemon functionality of the referenced EdgarRenderer.py and has no use of any
file system objects. It is thus adequately secure for deployment in a server environment
as no information from an invocation ever appears in the file system. (It would require
kernel access within the operating system to access insider information within processor
memory of a running instance.)
Modifications to the original EdgarRenderer source code are done under the Arelle(r)
license and protected by usual Arelle copyright and license. GitHub allows one to
inspect the changes log of the plugin branch from the SEC contributed branch to identify
modifications to the original.
This code is in gitHub/arelle/EdgarRenderer the edgr154 branch.
Installable applications are available with this module for Windows and MacOS on
http://arelle.org/download. The GUI versions run this code after selecting the
EdgarRenderer plugin (help->manage plugins, click select, select EdgarRenderer).
For command line operation, install arelle somewhere, and on Windows run arelleCmdLine.
On MacOS, at Arelle.app, with terminal shell, cd /Applications/Arelle.app/Contents/MacOS/,
and type in ./arelleCmdLine --about (or other parameters as needed).
To run from source code, libraries required are the following or newer:
pip install pytz-2014.10-py2.py3-none-any.whl
pip install six-1.9.0-py2.py3-none-any.whl
pip install python_dateutil-2.4.0-py2.py3-none-any.whl
pip install pyparsing-2.0.3-py3-none-any.whl
pip install numpy-1.9.1+mkl-cp34-none-win_amd64.whl
pip install matplotlib-1.4.2-cp34-none-win_amd64.whl
pip install jdcal-1.0-py2.py3-none-any.whl
pip install openpyxl-2.1.4-py2.py3-none-any.whl
To debug under eclipse from a normal eclipse project of Arelle it is suggested to check out
EdgarRenderer[edgr154] from GitHub under the arelle plugin directory (e.g., this file would be
plugin/EdgarRenderer/__init__.py). Note that EdgarGenderer is not part of Arelle itself,
it is in .gitignore in the top level Arelle directory.
Alternatively on a Mac or linux system one may soft link to the eclipse project's plugin
directory from a different directory containing a local copy of the EdgarRenderer plugin
project. Either way eclipse can debug and modify EdgarRenderer source code when debugging
and inspecting Arelle. On a Mac or Linux the soft link command would be:
ln -s ...EdgarRenderer (parent of this file) ...arelle/plugin (the eclipse project directory of plugin)
add ...arelle/plugin/EdgarRenderer to your .gitignore file
To run under in a server mode, with a single input zip (all in memory, not unpacked to
the file system) and a single output zip (all in memory, not on the file system):
a) when invoking via arelleCmdLine.py:
python3.4 arelleCmdLine.py (or arelleCmdLine, windows, or ./arelleCmdLine, Mac)
-f "/mydir/test/filingInstanceXsdAndLinkbases.zip"
-o "/mydir/test/out.zip"
--plugins EdgarRenderer # if installed in plugins, else full path to it: /mydir/myplugins/EdgarRenderer"
--disclosureSystem efm-pragmatic
--logFile mylogfilename.xxx
--debugMode
Specifying --logFile mylogfilename.txt (or .xml, .json) captures the log output in the
zip file with the specified file name. The suffix should be .xml or .json to get detailed logging parameters in
the respective format.
Adding --debugMode allows uncaught exceptions to provide a trace-back to eclipse, remove that
for production. Internet connectivity is by default offline at SEC, so override in this case.
If in a closed environment with all taxonomies in Taxonomy Packages or preloaded to cache, add
--internetConnectivity offline
b) when invoking via REST interface (built in webserver or cgi-bin server):
1) simple curl request or equivalent in code:
curl -X POST "-HContent-type: application/zip"
-T amd.zip
-o out.zip
"http://localhost:8080/rest/xbrl/validation?efm-pragmatic&media=zip&plugins=EdgarRenderer&logFile=log.xml"
(-T specifies zip of the filing, -o the file to store rendering results)
(use log.txt if a text log files is desired instead)
2) to not load EdgarRenderer dynamically, it must be active in plugins.json (as set up by GUI)
(sibling to the caches directoryexcept Mac where it's under ~/Library/Application Support/Arelle)
then omit &plugins=EdgarRenderer
the &logFile parameter specifies providing the log output in specified format in zip file (as noted above)
To run (as in EDGAR) with output report files added to the submission directory
python3.4 arelleCmdLine.py (or arelleCmdLine for Win/Mac apps)
-f "/mydir/test/amd.zip"
-r "/mydir/test" <<- the submission + output reports directory
--logFile logToBuffer or an specify an xml log file <<- required to save log messages into filing summary
--plugins 'EdgarRenderer' # if installed in plugins, else full path to it: /mydir/myplugins/EdgarRenderer"
--disclosureSystem efm-pragmatic
Note that the -r "out" directory is cleared on each run to assure no prior run files get inter
The filename parameter (-f) may be a JSON structure (as used in EDGAR itself) to pass in
more information about the filing (see validate/EFM/__init__.py) such as:
for a single instance filing, such as an SDR K form (without \n's pretty printed below):
-f '[{"file": "/Users/joe/.../gpc_gd1-20130930.htm",
"cik": "0000350001",
"cikNameList": {"0001306276":"BIGS FUND TRUST CO"},
"submissionType":"SDR-A",
"attachmentDocumentType":"EX-99.K SDR.INS",
"accessionNumber":"0001125840-15-000159"}]'
for multiple instances (SDR-L's), add more of the {"file": ...} entries.
for Windows called from Java, the JSON must be quoted as thus:
-f "[{\"file\":\"z:\\Documents\\...\\gpc_gd1-20130930.htm\",
\"cik\": \"0000350001\",
\"cikNameList\": {\"0000350001\":\"BIG FUND TRUST CO\"},
\"submissionType\":\"SDR-A\", \"attachmentDocumentType\":\"EX-99.K SDR.INS\"}]"
To build an installable cx_Freeze binary, (tested on Ubuntu), uncomment the entries in Arelle's
setup.py that are marked for EdgarRenderer.
Required if running under Java (using runtime.exec) on Windows, suggested always:
if xdgConfigHome or environment variable XDG_CONFIG_HOME are set:
please set environment variable MPLCONFIGDIR to same location as xdgConfigHome/XDG_CONFIG_HOME
(to prevent matlib crash under runtime.exe with Java)
Language of labels:
Multi-language labels will select first the label language, secondly en-US if different from label language, and lastly qname.
Command line (and Web API) may override system language for labels with parameter --labelLang
GUI may use tools->language labels setting to override system language for labels
"""
VERSION = '3.24.2'
from collections import defaultdict
from arelle import PythonUtil
from arelle import (Cntlr, FileSource, ModelDocument, XmlUtil, Version, ModelValue, Locale, PluginManager, WebCache, ModelFormulaObject, Validate,
ViewFileFactList, ViewFileFactTable, ViewFileConcepts, ViewFileFormulae,
ViewFileRelationshipSet, ViewFileTests, ViewFileRssFeed, ViewFileRoleTypes)
from arelle.ModelInstanceObject import ModelFact, ModelInlineFootnote
from arelle.PluginManager import pluginClassMethods
from arelle.ValidateFilingText import elementsWithNoContent
from arelle.XhtmlValidate import xhtmlValidate
from arelle.XmlValidateConst import VALID, NONE, UNVALIDATED
from arelle.XmlValidate import validate as xmlValidate
from . import RefManager, IoManager, Inline, Utils, Filing, Summary
import datetime, zipfile, logging, shutil, gettext, time, shlex, sys, traceback, linecache, os, io, tempfile
import regex as re
from lxml import etree
from os import getcwd, remove, removedirs
from os.path import join, isfile, exists, dirname, basename, isdir
from optparse import OptionParser, SUPPRESS_HELP
MODULENAME = os.path.basename(os.path.dirname(__file__))
tagsWithNoContent = set(f"{{http://www.w3.org/1999/xhtml}}{t}" for t in elementsWithNoContent)
for t in ("schemaRef", "linkbaseRef", "roleRef", "arcroleRef", "loc", "arc"):
tagsWithNoContent.add(f"{{http://www.xbrl.org/2003/linkbase}}{t}")
tagsWithNoContent.add("{http://www.xbrl.org/2013/inlineXBRL}relationship")
def uncloseSelfClosedTags(doc):
doc.parser.set_element_class_lookup(None) # modelXbrl class features are already closed now, block class lookup
for e in doc.xmlRootElement.iter():
# check if no text, no children and not self-closable element for EDGAR
if (e.text is None and (not e.getchildren())
and e.tag not in tagsWithNoContent
# also skip ix elements which are nil
and not (e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true","1") and e.tag.startswith("{http://www.xbrl.org/2013/inlineXBRL}"))):
e.text = "" # prevents self-closing tag with etree.tostring for zip and dissem folders
def allowableBytesForEdgar(bytestr):
# encode xml-legal ascii bytes not acceptable to EDGAR
return re.sub(b"[\\^\x7F]", lambda m: b"&#x%X;" % ord(m[0]), bytestr)
def serializeXml(xmlRootElement):
initialComment = b'' # tostring drops initial comments
node = xmlRootElement
while node.getprevious() is not None:
node = node.getprevious()
if isinstance(node, etree._Comment):
initialComment = etree.tostring(node, encoding="ASCII") + b'\n' + initialComment
serXml = etree.tostring(xmlRootElement, encoding="ASCII", xml_declaration=True)
if initialComment and serXml and serXml.startswith(b"<?"):
i = serXml.find(b"<html")
if i:
serXml = serXml[:i] + initialComment + serXml[i:]
return allowableBytesForEdgar(serXml)
###############
logParamEscapePattern = re.compile("[{]([^}]*)[}](?![}])") # match a java {{xxx} pattern to connvert to python escaped {{xxx}}
def edgarRendererCmdLineOptionExtender(parser, *args, **kwargs):
parser.add_option("-o", "--output", dest="zipOutputFile",
help=_("Zip the artifacts generated by the rendering process into a file with this name."))
parser.add_option("-c", "--configFile", dest="configFile",
default=os.path.join(os.path.dirname(__file__), 'conf', 'config_for_instance.xml'),
help=_("Path of location of Edgar Renderer configuration file relative to CWD. Default is EdgarRenderer.xml."))
parser.add_option("-r", "--reports", dest="reportsFolder",
help=_("Relative or absolute path and name of the reports folder."))
parser.add_option("--noReportOutput", action="store_true", dest="noReportOutput",
help=_("Block production of reports (FilingSummary, R files, xslx, etc)."))
parser.add_option("--resources", dest="resourcesFolder",
help=_("Relative path and name of the resources folder, which includes static xslt, css and js support files."))
parser.add_option("--filings", dest="filingsFolder",
help=_("Relative path and name of the filings (input) folder."))
parser.add_option("--processing", dest="processingFolder",
help=_("Relative path and name of the processing folder."))
parser.add_option("--errors", dest="errorsFolder",
help=_("Relative path and name of the errors folder, where unprocessable filings are stored."))
parser.add_option("--delivery", dest="deliveryFolder",
help=_("Relative path and name of the delivery folder, where all rendered artifacts are stored."))
parser.add_option("--archive", dest="archiveFolder",
help=_("Relative path and name of the archive folder, where successfully-processed original filings are stored."))
parser.add_option("--processingfrequency", dest="processingFrequency",
help=_("The sleep time for the RE3 daemon shell if no XBRL filing is present in the staging area."))
parser.add_option("--renderingService", dest="renderingService",
help=_("Type of service...Instance: one time rendering, or Daemon: background processing."))
parser.add_option("--totalClean", dest="totalClean", action="store_true",
help=_("Boolean to indicate if the shell should completely clean the archive, errors and delivery folders before processing."))
parser.add_option("--deleteProcessedFilings", dest="deleteProcessedFilings", action="store_true",
help=_("Boolean to indicate if processed filings should be deleted or not."))
parser.add_option("--htmlReportFormat", dest="htmlReportFormat",
help=_("Type of HTML report...Complete: asPage rendering = True, or Fragment: asPage rendering = False."))
parser.add_option("--reportFormat", dest="reportFormat",
help=_("One of Xml, Html, HtmlAndXml or None."))
parser.add_option("--failFile", dest="failFile", help=_("Relative path and name of fail file. "))
parser.add_option("--reportXslt", dest="reportXslt", help=_("Path and name of Stylesheet for producing a report file."))
parser.add_option("--reportXsltDissem", dest="reportXsltDissem", help=_("Path and name of Stylesheet for producing second report file for dissemination."))
parser.add_option("--summaryXslt", dest="summaryXslt", help=_("Path and name of Stylesheet, if any, for producing filing summary html."))
parser.add_option("--summaryXsltDissem", dest="summaryXsltDissem", help=_("Path and name of Stylesheet, if any, for producing second filing summary html for dissemination."))
parser.add_option("--renderingLogsXslt", dest="renderingLogsXslt", help=_("Path and name of Stylesheet, if any, for producing filing rendering logs html."))
parser.add_option("--excelXslt", dest="excelXslt", help=_("Path and name of Stylesheet, if any, for producing Excel 2007 xlsx output."))
parser.add_option("--auxMetadata", action="store_true", dest="auxMetadata", help=_("Set flag to generate inline xbrl auxiliary files"))
# saveTarget* parameters are added by inlineXbrlDocumentSet.py plugin
parser.add_option("--sourceList", action="store", dest="sourceList", help=_("Comma-separated triples of instance file, doc type and source file."))
parser.add_option("--copyInlineFilesToOutput", action="store_true", dest="copyInlineFilesToOutput", help=_("Set flag to copy all inline files to the output folder or zip."))
parser.add_option("--copyXbrlFilesToOutput", action="store_true", dest="copyXbrlFilesToOutput", help=_("Set flag to copy all source xbrl files to the output folder or zip."))
parser.add_option("--zipXbrlFilesToOutput", action="store_true", dest="zipXbrlFilesToOutput", help=_("Set flag to zip all source xbrl files to the an accession-number-xbrl.zip in reports folder or zip when an accession number parameter is available."))
parser.add_option("--includeLogsInSummary", action="store_true", dest="includeLogsInSummary", help=_("Set flag to copy log entries into <logs> in FilingSummary.xml."))
parser.add_option("--includeLogsInSummaryDissem", action="store_true", dest="includeLogsInSummaryDissem", help=_("Set flag to copy log entries into <logs> in FilingSummary.xml for dissemination."))
parser.add_option("--noLogsInSummary", action="store_false", dest="includeLogsInSummary", help=_("Unset flag to copy log entries into <logs> in FilingSummary.xml."))
parser.add_option("--noLogsInSummaryDissem", action="store_false", dest="includeLogsInSummaryDissem", help=_("Unset flag to copy log entries into <logs> in FilingSummary.xml for dissemination."))
parser.add_option("--processXsltInBrowser", action="store_true", dest="processXsltInBrowser", help=_("Set flag to process XSLT transformation in browser (e.g., rendering logs)."))
parser.add_option("--noXsltInBrowser", action="store_false", dest="processXsltInBrowser", help=_("Unset flag to process XSLT transformation in browser (e.g., rendering logs)."))
parser.add_option("--noEquity", action="store_true", dest="noEquity", help=_("Set flag to suppress special treatment of Equity Statements. "))
parser.add_option("--showErrors", action="store_true", dest="showErrors",
help=_("List all errors and warnings that may occur during RE3 processing."))
parser.add_option("--debugMode", action="store_true", dest="debugMode", help=_("Let the debugger handle exceptions."))
parser.add_option("--logMessageTextFile", action="store", dest="logMessageTextFile", help=_("Log message text file."))
# always use a buffering log handler (even if file or std out)
parser.add_option("--logToBuffer", action="store_true", dest="logToBuffer", default=True, help=SUPPRESS_HELP)
parser.add_option("--noRenderingWithError", action="store_true", dest="noRenderingWithError", help=_("Prevent rendering action when exhibit instance validation encountered error(s), blocking R file and extracted xml instance generation for that exhibit instance."))
parser.add_option("--keepFilingOpen", dest="keepFilingOpen", action="store_true", help=SUPPRESS_HELP) # block closing filing in filingEnd
class EdgarRenderer(Cntlr.Cntlr):
"""
.. class:: EdgarRenderer()
Initialization sets up for platform via Cntlr.Cntlr.
"""
def __init__(self, cntlr):
self.VERSION = VERSION
self.cntlr = cntlr
self.webCache = cntlr.webCache
self.preloadedPlugins = {}
self.ErrorMsgs = []
self.entrypoint = None # Contains the absolute path of the instance, inline, zip, or folder.
self.entrypointFolder = None # Contains absolute folder of instance, inline, or zip; equal to entrypoint if a folder.
self.nextFileNum = 1 # important for naming file numbers for multi-instance filings
self.nextUncategorizedFileNum = 9999
self.nextBarChartFileNum = 0
self.xlWriter = None
self.excelXslt = None
self.createdFolders = []
self.success = True
self.labelLangs = ['en-US','en-GB'] # list by WcH 7/14/2017, priority of label langs, en-XX always falls back to en anyway
if not hasattr(cntlr, "editedIxDocs"): # in GUI mode may be initialized in 'ModelDocument.Discover' before this class init
cntlr.editedIxDocs = {}
cntlr.editedModelXbrls = set()
cntlr.redlineIxDocs = {}
cntlr.redactTgtElts = set()
cntlr.redactTgtEltContent = set()
# iXBRLViewer plugin is present if there's a generate method
self.hasIXBRLViewer = any(True for generate in pluginClassMethods("iXBRLViewer.Generate"))
# wrap controler properties as needed
@property
def modelManager(self):
return self.cntlr.modelManager
@property
def logger(self):
return self.cntlr.logger
def processShowOptions(self, options):
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.logInfo(_("Option {0}={1}").format(optName, optValue))
IoManager.logConfigFile(self, options, messageCode='info')
def retrieveDefaultREConfigParams(self, options):
# defaultValueDict contains the default strings for
# when there are no config file and no command line arguments.
self.defaultValueDict = defaultdict(lambda:None)
self.defaultValueDict['abortOnMajorError'] = str(False)
self.defaultValueDict['noRenderingWithError'] = str(False)
self.defaultValueDict['archiveFolder'] = 'Archive'
self.defaultValueDict['auxMetadata'] = str(False)
self.defaultValueDict['copyInlineFilesToOutput'] = str(False)
self.defaultValueDict['copyXbrlFilesToOutput'] = str(False)
self.defaultValueDict['zipXbrlFilesToOutput'] = str(False)
self.defaultValueDict['includeLogsInSummary'] = str(False)
self.defaultValueDict['includeLogsInSummaryDissem'] = str(False)
self.defaultValueDict['deleteProcessedFilings'] = str(True)
self.defaultValueDict['deliveryFolder'] = 'Delivery'
self.defaultValueDict['debugMode'] = str(False)
self.defaultValueDict['errorsFolder'] = 'Errors'
self.defaultValueDict['excelXslt'] = 'InstanceReport_XmlWorkbook.xslt'
self.defaultValueDict['failFile'] = 'errorLog.log'
self.defaultValueDict['filingsFolder'] = 'Filings'
self.defaultValueDict['htmlReportFormat'] = 'Complete'
self.defaultValueDict['internetConnectivity'] = 'online' # HF change to online default 'offline'
self.defaultValueDict['noEquity'] = str(False)
self.defaultValueDict['processingFolder'] = 'Processing'
self.defaultValueDict['processingFrequency'] = '10'
self.defaultValueDict['processXsltInBrowser'] = str(False)
self.defaultValueDict['renderingLogsXslt'] = None
self.defaultValueDict['renderingService'] = 'Instance'
self.defaultValueDict['reportFormat'] = 'Html'
self.defaultValueDict['reportsFolder'] = 'Reports'
self.defaultValueDict['reportXslt'] = 'InstanceReport.xslt'
self.defaultValueDict['reportXsltDissem'] = None
self.defaultValueDict['resourcesFolder'] = "resources"
self.defaultValueDict['saveTargetInstance'] = str(True)
self.defaultValueDict['saveTargetFiling'] = str(False)
self.defaultValueDict['sourceList'] = ''
self.defaultValueDict['summaryXslt'] = None
self.defaultValueDict['summaryXsltDissem'] = None
self.defaultValueDict['totalClean'] = str(False)
self.defaultValueDict['utrValidate'] = str(False)
self.defaultValueDict['validate'] = str(False)
self.defaultValueDict['validateEFM'] = str(False)
self.defaultValueDict['zipOutputFile'] = None
self.defaultValueDict['logMessageTextFile'] = None
# The configDict holds the values as they were read from the config file.
# Only options that appear with a value that is not None in defaultValueDict are recognized.
self.configDict = defaultdict(lambda:None)
configLocation = IoManager.getConfigFile(self,options)
if configLocation is None: # Although it is odd not to have a config file, it is not an error.
self.logDebug(_("No config file"))
else:
self.logDebug(_("Extracting info from config file {}".format(os.path.basename(configLocation))))
tree = etree.parse(configLocation)
for child in tree.iter():
if child.tag is not etree.Comment and child.text is not None:
if child.tag in self.defaultValueDict:
value = child.text.strip()
if value == '': value = None
self.configDict[child.tag] = value
elif len(child.text.strip()) > 0: # Extra tags with no content are ignored
#message = ErrorMgr.getError('UNSUPPORTED_CONFIG_TAG').format(child.text, child.tag)
self.logWarn("Found value {} for unsupported configuration tag {}".format(child.text, child.tag))
def initializeReOptions(self, options, setCntlrOptions=False):
self.logDebug("General options:")
def setProp(prop, init, rangeList=None, cs=False):
value = next((x
for x in [init, self.configDict[prop], self.defaultValueDict[prop]]
if x is not None), None)
if type(value)==str and not cs: value = value.casefold()
setattr(self, prop, value)
if rangeList is not None and value not in [x.casefold() for x in rangeList]:
raise Exception("Unknown {} '{}' not in {} (case insensitive) on command line or config file.".format(prop,value,rangeList))
self.logDebug("{}=\t{}".format(prop, value))
return value
# options applicable to rendering in either mode:
options.renderingService = setProp('renderingService', options.renderingService, rangeList=['Instance','Daemon'])
options.reportFormat = setProp('reportFormat', options.reportFormat, rangeList=['Html', 'Xml', 'HtmlAndXml', 'None'])
options.htmlReportFormat = setProp('htmlReportFormat', options.htmlReportFormat, rangeList=['Complete','Fragment'])
options.zipOutputFile = setProp('zipOutputFile', options.zipOutputFile,cs=True)
options.sourceList = " ".join(setProp('sourceList', options.sourceList,cs=True).split()).split(',')
self.sourceDict={}
# Parse comma and colon separated list a:b b:c, d:e:f into a dictionary {'a': ('b b','c'), 'd': ('e','f') }:
for source in options.sourceList:
if (len(source) > 0):
s = source.split(':') # we must accommodate spaces in tokens separated by colons
if (len(s) != 3):
self.logWarn("Ignoring bad token {} in {}".format(s,options.sourceList))
else: # we do not accommodate general URL's in the 'original' field.
instance = " ".join(s[0].split())
doctype = " ".join(s[1].split())
original = " ".join(s[2].split())
self.sourceDict[instance]=(doctype,original)
# These options have to be passed back to arelle via the options object
# HF, removed: options.internetConnectivity = setProp('internetConnectivity',options.internetConnectivity, rangeList=['online','offline'])
def setFlag(flag, init):
setattr(self, flag, next((Utils.booleanFromString(x)
for x in [init
, self.configDict[flag], self.defaultValueDict[flag]]
if x is not None), None))
self.logDebug("{}=\t{}".format(flag, getattr(self, flag)))
return getattr(self, flag)
# inherited flag: options.abortOnMajorError = setFlag('abortOnMajorError', options.abortOnMajorError)
setFlag('abortOnMajorError', options.abortOnMajorError)
setFlag('noRenderingWithError', options.noRenderingWithError)
options.totalClean = setFlag('totalClean', options.totalClean)
options.noEquity = setFlag('noEquity', options.noEquity)
options.auxMetadata = setFlag('auxMetadata', options.auxMetadata)
options.copyInlineFilesToOutput = setFlag('copyInlineFilesToOutput', options.copyInlineFilesToOutput)
options.copyXbrlFilesToOutput = setFlag('copyXbrlFilesToOutput', options.copyXbrlFilesToOutput)
options.zipXbrlFilesToOutput = setFlag('zipXbrlFilesToOutput', options.zipXbrlFilesToOutput)
options.includeLogsInSummary = setFlag('includeLogsInSummary', options.includeLogsInSummary)
options.includeLogsInSummaryDissem = setFlag('includeLogsInSummaryDissem', options.includeLogsInSummaryDissem)
options.processXsltInBrowser = setFlag('processXsltInBrowser', options.processXsltInBrowser)
self.summaryHasLogEntries = False
options.saveTargetInstance = setFlag('saveTargetInstance',options.saveTargetInstance)
options.saveTargetFiling = setFlag('saveTargetFiling',options.saveTargetFiling)
# note that delete processed filings is only relevant when the input had to be unzipped.
options.deleteProcessedFilings = setFlag('deleteProcessedFilings', options.deleteProcessedFilings)
options.debugMode = setFlag('debugMode', options.debugMode)
# These flags have to be passed back to arelle via the options object.
# inherited flag: options.validate = setFlag('validate', options.validate)
setFlag('validate', options.validate)
# inherited flag: options.utrValidate = setFlag('utrValidate', options.utrValidate)
setFlag('utrValidate', options.utrValidate)
# inherited flag: options.validateEFM = setFlag('validateEFM', options.validateEFM)
setFlag('validateEFM', options.validateEFM)
if setCntlrOptions:
# cmd line mode, pass back flags as set here
options.validate = self.validate
options.utrValidate = self.utrValidate
options.validateEFM = self.validateEFM
def setFolder(folder, init, searchPythonPath=False):
if searchPythonPath: # if the folder is not an absolute path, we want to look for it relative to the python path.
value= next((IoManager.absPathOnPythonPath(self,x)
for x in [init, self.configDict[folder], self.defaultValueDict[folder]]
if x is not None), None)
else: # otherwise interpret the folder as being relative to some subsequent processing location.
value = next((x
for x in [init, self.configDict[folder], self.defaultValueDict[folder]]
if x is not None), None)
setattr(self, folder, value)
self.logDebug("{}=\t{}".format(folder, getattr(self, folder)))
return getattr(self, folder)
options.processingFolder = setFolder('processingFolder', options.processingFolder)
self.processInZip = True # bool(options.processingFolder)
if options.noReportOutput:
options.reportsFolder = self.reportsFolder = self.initialReportsFolder = None
else:
options.reportsFolder = setFolder('reportsFolder', options.reportsFolder)
# keep initial reports folder, may be relative, if so reset on each new filing
self.initialReportsFolder = self.reportsFolder
self.reportInZip = True # bool(options.reportsFolder)
options.resourcesFolder = setFolder('resourcesFolder', options.resourcesFolder,searchPythonPath=True)
def setResourceFile(file, init, errstr):
setattr(self, file, None)
value = next((x
for x in [init, self.configDict[file], self.defaultValueDict[file]]
if x is not None), None)
if value is not None and type(value)==str and len(value)>0:
if not isdir(self.resourcesFolder):
# It is possible to specify a bad resources folder, but then not actually need it;
# that is why the test for its presence is here and not earlier.
raise Exception("Cannot locate resources folder '{}'".format(self.resourcesFolder))
value = os.path.join(self.resourcesFolder,value)
setattr(self, file, value)
if value is not None and not isfile(value):
raise Exception(_(errstr).format(value))
self.logDebug("{}=\t{}".format(file, value))
return value
#setResourceFile('reportXslt', options.reportXslt, 'INVALID_CONFIG_REPORTXSLT')
options.reportXslt = setResourceFile('reportXslt', options.reportXslt, "Cannot find report xslt {}")
options.reportXsltDissem = setResourceFile('reportXsltDissem', options.reportXsltDissem, "Cannot find dissemination report xslt {}")
# Report XSLT is required when reportFormat contains 'Html'.
if self.reportXslt is None and 'html' in self.reportFormat.casefold():
raise Exception('No {} specified when {}={} requires it.'.format('reportXslt', 'reportFormat', self.reportFormat))
# Summary XSLT is optional, but do report if you can't find it.
#setResourceFile('summaryXslt', options.summaryXslt, 'INVALID_CONFIG_SUMMARYXSLT')
options.summaryXslt = setResourceFile('summaryXslt', options.summaryXslt, "Cannot find summary xslt {}")
options.summaryXsltDissem = setResourceFile('summaryXsltDissem', options.summaryXsltDissem, "Cannot find dissemination summary xslt {}")
options.renderingLogsXslt = setResourceFile('renderingLogsXslt', options.renderingLogsXslt, "Cannot find rendering logs xslt {}")
# Excel XSLT is optional, but do report if you can't find it.
#setResourceFile('excelXslt', options.excelXslt, 'INVALID_CONFIG_EXCELXSLT')
# if options.excelXslt is "True" from web interface, it has no no string value, e.g., &excelXslt (looks like True here)
options.excelXslt = setResourceFile('excelXslt', "" if options.excelXslt == True else options.excelXslt, "Cannot find excel xslt {}")
# logMessageTextFile is optional resources file for messages text (SEC format)
options.logMessageTextFile = setResourceFile('logMessageTextFile', options.logMessageTextFile, "Cannot find logMessageTextFile {}")
_lang = options.labelLang or self.modelManager.defaultLang
if _lang in self.labelLangs:
self.labelLangs.remove(_lang)
self.labelLangs.insert(0, _lang) # ensure system lang is at start of langs list (highest priority)
# modelXbrl's must be closed by filingEnd
options.keepOpen = True
def copyReAttrOptions(self, options):
# set EdgarRenderer options from prior processed options object
self.renderingService = options.renderingService
self.reportFormat = options.reportFormat
self.htmlReportFormat = options.htmlReportFormat
self.zipOutputFile = options.zipOutputFile
self.sourceList = options.sourceList
self.sourceDict={}
# Parse comma and colon separated list a:b b:c, d:e:f into a dictionary {'a': ('b b','c'), 'd': ('e','f') }:
for source in options.sourceList:
if (len(source) > 0):
s = source.split(':') # we must accommodate spaces in tokens separated by colons
if (len(s) != 3):
self.logWarn("Ignoring bad token {} in {}".format(s,options.sourceList))
else: # we do not accommodate general URL's in the 'original' field.
instance = " ".join(s[0].split())
doctype = " ".join(s[1].split())
original = " ".join(s[2].split())
self.sourceDict[instance]=(doctype,original)
# These options have to be passed back to arelle via the options object
# HF, removed: options.internetConnectivity = setProp('internetConnectivity',options.internetConnectivity, rangeList=['online','offline'])
# inherited flag: options.abortOnMajorError = setFlag('abortOnMajorError', options.abortOnMajorError)
self.abortOnMajorError = options.abortOnMajorError
self.noRenderingWithError = options.noRenderingWithError
self.totalClean = options.totalClean
self.noEquity = options.noEquity
self.auxMetadata = options.auxMetadata
self.copyInlineFilesToOutput = options.copyInlineFilesToOutput
self.copyXbrlFilesToOutput = options.copyXbrlFilesToOutput
self.zipXbrlFilesToOutput = options.zipXbrlFilesToOutput
self.includeLogsInSummary = options.includeLogsInSummary
self.includeLogsInSummaryDissem = options.includeLogsInSummaryDissem
self.processXsltInBrowser = options.processXsltInBrowser
self.summaryHasLogEntries = False
self.saveTargetInstance = options.saveTargetInstance
self.saveTargetFiling = options.saveTargetFiling
# note that delete processed filings is only relevant when the input had to be unzipped.
self.deleteProcessedFilings = options.deleteProcessedFilings
self.debugMode = options.debugMode
# These flags have to be passed back to arelle via the options object.
# inherited flag: options.validate = setFlag('validate', options.validate)
self.validate = options.validate
# inherited flag: options.utrValidate = setFlag('utrValidate', options.utrValidate)
self.utrValidate = options.utrValidate
# inherited flag: options.validateEFM = setFlag('validateEFM', options.validateEFM)
self.validateEFM = options.validateEFM
self.processingFolder = options.processingFolder
self.processInZip = True # bool(options.processingFolder)
self.reportsFolder = options.reportsFolder # dynamically changed to relative of processing instance unless absolute
# keep initial reports folder, may be relative, if so reset on each new filing
self.initialReportsFolder = self.reportsFolder
self.reportInZip = True # bool(options.reportsFolder)
self.resourcesFolder = options.resourcesFolder
#setResourceFile('reportXslt', options.reportXslt, 'INVALID_CONFIG_REPORTXSLT')
self.reportXslt = options.reportXslt
self.reportXsltDissem = options.reportXsltDissem
# Report XSLT is required when reportFormat contains 'Html'.
if self.reportXslt is None and 'html' in self.reportFormat.casefold():
raise Exception('No {} specified when {}={} requires it.'.format('reportXslt', 'reportFormat', self.reportFormat))
# Summary XSLT is optional, but do report if you can't find it.
#setResourceFile('summaryXslt', options.summaryXslt, 'INVALID_CONFIG_SUMMARYXSLT')
self.summaryXslt = options.summaryXslt
self.summaryXsltDissem = options.summaryXsltDissem
self.renderingLogsXslt = options.renderingLogsXslt
# Excel XSLT is optional, but do report if you can't find it.
#setResourceFile('excelXslt', options.excelXslt, 'INVALID_CONFIG_EXCELXSLT')
self.excelXslt = options.excelXslt
self.logMessageTextFile = options.logMessageTextFile
if self.isDaemon:
self.initializeReDaemonOptions(options)
def initializeReSinglesOptions(self, options):
# At the moment there are not any options relevant only for single-instance mode.
return
def initializeReDaemonOptions(self, options):
self.logDebug("Daemon Options:")
def setLocation(location, init, mustExist=False, isfile=False):
value = next((os.path.join(os.getcwd(), x)
for x in [init
, self.configDict[location], self.defaultValueDict[location]]
if x is not None), None)
setattr(self, location, value)
self.logDebug("{}=\t{}".format(location, value))
if mustExist and value is None:
raise Exception('No {} specified for Daemon.'.format(location))
if mustExist and not isfile and not isdir(value):
raise Exception('No such {} as {}.'.format(location, value))
return value
setLocation('filingsFolder', options.filingsFolder)
setLocation('deliveryFolder', options.deliveryFolder)
setLocation('errorsFolder', options.errorsFolder)
setLocation('archiveFolder', options.archiveFolder)
setLocation('failFile', options.failFile)
def setProp(prop, init, required=False):
value = next((x
for x in [init, self.configDict[prop], self.defaultValueDict[prop]]
if x is not None), None)
setattr(self, prop, value)
if required and value is None:
raise Exception('{} not set for daemon'.format(prop))
self.logDebug("{}=\t{}".format(prop, getattr(self, prop)))
return value
setProp('processingFrequency', options.processingFrequency, required=True)
@property
def renderMode(self):
return self.renderingService.casefold()
@property
def isSingles(self):
return self.renderingService.casefold() == 'instance'
@property
def isDaemon(self):
return self.renderingService.casefold() == 'daemon'
def daemonDequeueInputZip(self, options): # returns the location of zip ready to unpack
# upon exit, options.entrypoint is set to absolute location of the zip file after the move,
# and self.processingFolder is set to absolute location of where it should be processed.
inputFileSource = None
zipfound = False
options.entrypointFile = None # no entrypoint file(s)
# check for next filing in input folder
# as usual, self.{some}Folder is absolute, while options.{some}Folder is what was specified in the input.
self.originalProcessingFolder = os.path.join(getcwd(), self.processingFolder)
self.logDebug(_("Checking for the oldest zip file in {}").format(self.filingsFolder))
while not zipfound:
for file in sorted(os.listdir(self.filingsFolder), key=lambda file: os.stat(join(self.filingsFolder, file)).st_mtime):
if not zipfound and Utils.isZipFilename(file):
inputFileSource = join(self.filingsFolder, file)
self.processingFolder = IoManager.createNewFolder(self,self.originalProcessingFolder,file)
# reportsFolder = normpath(join(self.processingFolder,options.reportsFolder))
processingFileSource = join(self.processingFolder, file)
if not exists(inputFileSource): continue # it got deleted before we could process it.
self.logDebug(_("Found a new zip file to process; moving {} to Processing folder ").format(inputFileSource))
try:
IoManager.move_clobbering_file(inputFileSource, processingFileSource)
options.entrypointFile = processingFileSource
zipfound = True
except IOError as err:
self.logError(str(err))
#self.logError(_(ErrorMgr.getError('FILING_MOVE_ERROR').format(self.processingFolder)))
self.logError(_("Could not remove {}").format(self.processingFolder))
try: removedirs(self.processingFolder)
except IOError: continue
# provide these parameters to FilingStart via options
options.zipOutputFile = join(self.processingFolder, "-out".join(os.path.splitext(file)))
options.doneFile = join(self.archiveFolder, file)
# self.failFile = join(self.errorsFolder,file)
if self.createdFolders: # pass to filing processing any created folders
options.daemonCreatedFolders = self.createdFolders
break
# no more files.
if not zipfound:
sleep = self.processingFrequency
self.logDebug(_("Sleeping for " + sleep + " seconds. "))
time.sleep(float(sleep))
self.cntlr.logHandler.flush() # flush log buffer (if any)
return
def setProcessingFolder(self, filesource, entryPointFile=None):
if filesource and self.processingFolder == self.defaultValueDict['processingFolder']:
if filesource.isOpen:
self.processingFolder = os.path.dirname(filesource.basefile)
else:
_url = filesource.url
#filesource.url may have an inline document set, trim it off
for pluginXbrlMethod in pluginClassMethods("InlineDocumentSet.Url.Separator"):
inlineDocSetSeparator = pluginXbrlMethod()
_url = _url.partition(inlineDocSetSeparator)[0]
self.processingFolder = os.path.dirname(_url)
if entryPointFile and os.path.exists(entryPointFile): # for testcase, is different than filesource, which points to testcase
if not (filesource and filesource.isOpen and filesource.isArchive and entryPointFile.startswith(filesource.basefile)):
self.processingFolder = os.path.dirname(entryPointFile)
def checkIfDaemonStartup(self, options):
# startup (when in Deamon mode)
self.retrieveDefaultREConfigParams(options)
self.initializeReOptions(options, setCntlrOptions=True)
# if isDaemon, wait for an input zip to process
# if not isDaemon, then instance (or zip) is ready to process immediately
if self.isDaemon:
self.initializeReDaemonOptions(options)
IoManager.handleFolder(self, self.filingsFolder, False, False)
IoManager.handleFolder(self, self.deliveryFolder, False, self.totalClean)
IoManager.handleFolder(self, self.archiveFolder, False, self.totalClean)
if self.errorsFolder is not None: # You might not have an errors folder.
IoManager.handleFolder(self, self.errorsFolder, False, self.totalClean)
self.daemonDequeueInputZip(options) # loop, waiting for an input to process, then returns and processes input as if --file specified the input
def isRunningUnderTestcase(self):
return (getattr(self.cntlr.modelManager.loadedModelXbrls[0], "modelDocument", None) is not None and
self.cntlr.modelManager.loadedModelXbrls[0].modelDocument.type in ModelDocument.Type.TESTCASETYPES)
def filingStart(self, cntlr, options, entrypointFiles, filing):
# start a (mult-iinstance) filing
filing.edgarRenderer = self
self.reportZip = filing.reportZip
self.writeFile = filing.writeFile
# Set default config params; overwrite with command line args if necessary
self.retrieveDefaultREConfigParams(options)
# Initialize the folders and objects required in both modes.
if options.sourceList is None: # options is none on first GUI run testcase variation
self.initializeReOptions(options)
else: # options previously initialized
self.copyReAttrOptions(options)
# Transfer daemonCreatedFilders to this EdgarRenderer to deal with at filingEnd
if hasattr(options, "daemonCreatedFolders"):
self.createdFolders.extend(options.daemonCreatedFolders)
del options.daemonCreatedFolders # don't pass to any subsequent independent filing if any
mdlMgr = cntlr.modelManager
self.disclosureSystem = mdlMgr.disclosureSystem
self.validatedForEFM = ("esef" in self.disclosureSystem.names or # prevent EFM validation messages for "esef" validations
(not cntlr.hasGui and mdlMgr.validateDisclosureSystem and getattr(mdlMgr.disclosureSystem, "EFMplugin", False))
)
self.instanceSummaryList = []
self.instanceList = []
self.inlineList = []
self.otherXbrlList = []
self.supplementList = []
self.supplementalFileList = []
self.renderedFiles = filing.renderedFiles # filing-level rendered files
self.setProcessingFolder(filing.filesource)
self.abortedDueToMajorError = False
if self.isDaemon:
self.zipOutputFile = options.zipOutputFile
self.doneFile = options.doneFile
self.isWorkstationFirstPass = os.path.split(self.reportXslt or "")[-1] == "EdgarWorkstationInstanceReport.xslt"
self.loopnum = 0
def processInstance(self, options, modelXbrl, filing, report):
# skip rendering if major errors and abortOnMajorError
# errorCountDuringValidation = len(Utils.xbrlErrors(modelXbrl))
# won't work for all possible logHandlers (some emit immediately)
attachmentDocumentType = getattr(modelXbrl, "efmIxdsType")
# strip on error if preceding primary inline instance had no error and exhibitType strips on error
stripExhibitOnError = self.success and bool(
filing.exhibitTypesStrippingOnErrorPattern.match(attachmentDocumentType or ""))
errorCountDuringValidation = sum(1 for e in modelXbrl.errors if isinstance(e, str) and not e.startswith("DQC.")) # don't count assertion results dict if formulas ran
success = True
if errorCountDuringValidation > 0:
if self.abortOnMajorError: # HF renderer raises exception on major errors: self.modelManager.abortOnMajorError:
self.logFatal(_("Not attempting to render after {} validation errors").format(
errorCountDuringValidation))
self.abortedDueToMajorError = True
return
else:
self.logInfo(_("Ignoring {} Validation errors because abortOnMajorError is not set.").format(errorCountDuringValidation))
if not self.isRunningUnderTestcase():
success = False
modelXbrl.profileActivity()
self.setProcessingFolder(modelXbrl.fileSource, report.filepaths[0]) # use first of possibly multi-doc IXDS files
# if not reportZip and reportsFolder is relative, make it relative to source file location (on first report)
if (success or not self.noRenderingWithError) and not filing.reportZip and self.initialReportsFolder and self.loopnum == 0:
if not os.path.isabs(self.initialReportsFolder):
# try input file's directory
if os.path.exists(self.processingFolder) and os.access(self.processingFolder, os.W_OK | os.X_OK):
self.reportsFolder = os.path.join(self.processingFolder, self.initialReportsFolder)
else:
_dir = modelXbrl.modelManager.cntlr.webCache.urlToCacheFilepath(self.processingFolder)
# try cache directory locating reportsFolder where the entry point instance is in cache
if os.path.exists(_dir) and os.access(_dir, os.W_OK | os.X_OK):
self.reportsFolder = os.path.join(_dir, self.initialReportsFolder)
else: # a temp directory
self.reportsFolder = tempfile.mkdtemp(prefix="EdgarRenderer_") # Mac or Windows temp directory
IoManager.handleFolder(self, self.reportsFolder, True, self.totalClean)
self.renderedFiles = report.renderedFiles # report-level rendered files
RefManager.RefManager(self.resourcesFolder).loadAddedUrls(modelXbrl, self) # do this after validation.
self.loopnum += 1
try:
if success or not self.noRenderingWithError: # no instance errors from prior validation workflow
# recheck for errors
if (stripExhibitOnError and sum(1 for e in modelXbrl.errors if isinstance(e, str)) > 0):
success = False
self.logDebug(_("Stripping filing due to {} preceding validation errors.").format(errorCountDuringValidation))
if success or (not self.noRenderingWithError and not stripExhibitOnError):
# add missing IDs to inline documents
for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()):
doc = ixdsHtmlRootElt.modelDocument
hasIdAssignedFact = False
for e in ixdsHtmlRootElt.iter(doc.ixNStag + "nonNumeric", doc.ixNStag + "nonFraction", doc.ixNStag + "fraction"):
if getattr(e, "xValid", 0) >= VALID and not e.id: # id is optional on facts but required for ixviewer-plus and arelle inline viewers
id = f"ixv-{e.objectIndex}"
if id in doc.idObjects or id in modelXbrl.ixdsEltById:
for i in range(1000):
uid = f"{id}_{i}"
if uid not in doc.idObjects and uid not in modelXbrl.ixdsEltById:
id = uid
break
e.set("id", id)
doc.idObjects[id] = e
modelXbrl.ixdsEltById[id] = e
hasIdAssignedFact = True
if (hasIdAssignedFact or self.isWorkstationFirstPass) and self.reportsFolder:
self.cntlr.editedIxDocs[doc.basename] = doc # causes it to be rewritten out
self.cntlr.editedModelXbrls.add(modelXbrl)
except Utils.RenderingException as ex:
success = False # error message provided at source where exception was raised
self.logDebug(_("RenderingException after {} validation errors: {}").format(errorCountDuringValidation, ex))
except Exception as ex:
success = False
action = "complete validation" if options.noReportOutput else "produce output"
if errorCountDuringValidation > 0:
self.logWarn(_("The rendering engine was unable to {} after {} validation errors.").format(action, errorCountDuringValidation))
else:
self.logWarn(_("The rendering engine was unable to {} due to an internal error. This is not considered an error in the filing.").format(action, errorCountDuringValidation))
self.logDebug(_("Exception traceback: {}").format(traceback.format_exception(*sys.exc_info())))
if success or (not self.noRenderingWithError and not stripExhibitOnError):
for basename in report.basenames:
if report.isInline:
if basename not in self.inlineList:
self.inlineList.append(basename)
elif basename.endswith(".xml"):
if basename not in self.instanceList:
self.instanceList.append(basename)
for reportedFile in sorted(report.reportedFiles):
if Utils.isImageFilename(reportedFile):
self.supplementalFileList.append(reportedFile)
self.supplementList.append(reportedFile)
#elif reportedFile.endswith(".htm"): # the non-inline primary document isn't known to Arelle yet in EDGAR
# self.inlineList.append(reportedFile)
elif reportedFile not in report.basenames:
self.otherXbrlList.append(reportedFile)
self.renderedFiles = filing.renderedFiles # filing-level rendered files
if not success:
if stripExhibitOnError:
filing.reports.remove(report) # remove stripped report from filing (so it won't be in zip)
filing.strippedFiles[attachmentDocumentType] |= report.reportedFiles # strip files not referenced from unstripped reports
else:
self.success = False
# remove any inline invalid facts, assign and note if any missing IDs
for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()):
doc = ixdsHtmlRootElt.modelDocument
_ixHidden = doc.ixNStag + "hidden"
hasEditedFact = False
elementsToRemove = []
for e in ixdsHtmlRootElt.iter(doc.ixNStag + "nonNumeric", doc.ixNStag + "nonFraction", doc.ixNStag + "fraction"):
if getattr(e, "xValid", 0) < VALID:
e.set("title", f"Removed invalid ix:{e.tag.rpartition('}')[2]} element, fact {e.qname} contextId {e.contextID}")
for attr in e.keys():
if attr not in ("id", "title"):
e.attrib.pop(attr, None)
if e.getparent().tag == _ixHidden:
e.addprevious(etree.Comment(f"Removed invalid ix:{e.tag.rpartition('}')[2]} element, fact {e.qname} contextId {e.contextID}: \"{(e.text or '').replace('--','- -')}\""))
elementsToRemove.append(e)
elif (e.getparent().tag == "{http://www.w3.org/1999/xhtml}body" or
any(c.tag in ("{http://www.w3.org/1999/xhtml}table", "{http://www.w3.org/1999/xhtml}div")
for c in e.iterchildren())):
e.tag = "{http://www.w3.org/1999/xhtml}div"
else:
e.tag = "{http://www.w3.org/1999/xhtml}span"
hasEditedFact = True
for e in elementsToRemove: # remove ix hidden invalid elements
e.getparent().remove(e)
if hasEditedFact:
self.cntlr.editedIxDocs[doc.basename] = doc # causes it to be rewritten out
if getattr(doc, "securityClassification", None):
# note for SBSEF confidential reports have only one ix doc so whole report is treated confidential
report.securityClassification = doc.securityClassification
# block closing filesource when modelXbrl closes because it's used by filingEnd (and may be an archive)
modelXbrl.closeFileSource = False
modelXbrl.profileStat(_("EdgarRenderer process instance {}").format(report.basenames[0]))
def loadLogMessageText(self):
self.logMessageText = {}
if self.logMessageTextFile:
try:
for msgElt in etree.parse(self.logMessageTextFile).iter("message"):
self.logMessageText[msgElt.get("code")] = re.sub(
r"\$\((\w+)\)", r"%(\1)s", msgElt.text.strip())
except Exception as ex:
self.logDebug(_("Exception loading logMessageText file, traceback: {}").format(traceback.format_exception(*sys.exc_info())))
self.logMessageText.clear() # don't leave possibly erroneous messages text entries
def formatLogMessage(self, logRec):
logHandler = self.cntlr.logHandler
fileLines = defaultdict(set)
for ref in logRec.refs:
href = ref.get("href")
if href:
fileLines[href.partition("#")[0]].add(ref.get("sourceLine", 0))
_text = logHandler.format(logRec) # sets logRec.file
if hasattr(logHandler.formatter, "fileLines"):
_fileLines = logHandler.formatter.fileLines(logRec)
if _fileLines:
_text += " - " + _fileLines # default if no {refSources} or other in the ArelleMessagesText
try:
# try edgarCode and if not there, try messageCode
if isinstance(logRec.args,dict) and logRec.args.get("edgarCode") in self.logMessageText:
_msgText = self.logMessageText[logRec.args.get("edgarCode")]
elif logRec.messageCode in self.logMessageText:
_msgText = self.logMessageText[logRec.messageCode]
else:
return _text
_msgParams = logParamEscapePattern.findall(_msgText) # finds all parameters
_msgArgs = logRec.args.copy() # duplicate functinoality of ArelleMessageWrapper.java
_refNum = 0
_fileLines = defaultdict(set)
for _ref in logRec.refs:
_href = _ref.get("href")
if _href:
_hrefUrlParts = href.split("#")
if len(_hrefUrlParts) > 0:
_refNum += 1
_fileName = _hrefUrlParts[0].rpartition("/")[2]
_sourceLine = _ref.get("sourceLine")
if _sourceLine:
_source = "{} line {}".format(_fileName, _sourceLine)
if "refLine" not in _msgArgs:
_msgArgs["refLine"] = _sourceLine
else:
_source = _fileName
if "refSource" not in _msgArgs: # first only
_msgArgs["refSource"] = _source
_msgArgs["refUrl"] = _fileName
else:
_msgArgs["refSource{}".format(_refNum)] = _source
if "refSources2_n" in _msgArgs:
_msgArgs["refSources2_n"] = _msgArgs.get("refSources2_n") + ", " + _source
else:
_msgArgs["refSources2_n"] = _source
_fileLines[_fileName].add(_sourceLine)
if logRec.refs:
_msgArgs["refSources"] = Cntlr.logRefsFileLines(logRec.refs)
_text = logParamEscapePattern.sub(r"{\1}", # substitute java {{x} into py {{x}}} but leave {{x}} as it was
_msgText
).format(**_msgArgs) # now uses {...} parameters in arelleMessagesText
except KeyError:
pass # not replacable messageCode or a %(xxx)s format arg was not in the logRec arcs or it's a $() java function reference
except ValueError as err: # error inn { ... } parameters specification
self.logDebug("Message format string error {} in string {}".format(err, logRec.messageCode))
return _text
def transformFilingSummary(self, filing, rootETree, xsltFile, reportsFolder, htmFileName, includeLogs, title=None, zipDir=""):
summary_transform = etree.XSLT(etree.parse(xsltFile))
trargs = {"asPage": etree.XSLT.strparam('true'),
"accessionNumber": "'{}'".format(getattr(filing, "accessionNumber", "")),
"resourcesFolder": "'{}'".format(self.resourcesFolder.replace("\\","/")),
"processXsltInBrowser": etree.XSLT.strparam(str(self.processXsltInBrowser).lower()),
"includeLogs": etree.XSLT.strparam(str(includeLogs).lower()),
"includeExcel": etree.XSLT.strparam("true" if (self.excelXslt) else "false")}
if title:
trargs["title"] = etree.XSLT.strparam(title)
result = summary_transform(rootETree, **trargs)
IoManager.writeHtmlDoc(filing, result, self.reportZip, reportsFolder, htmFileName, zipDir);
def filingEnd(self, cntlr, options, filesource, filing, sourceZipStream=None, *args, **kwargs):
# note that filesource is None if there were no instances
if self.abortedDueToMajorError:
self.success = False # major errors detected