feat: Add option to update datatracker references (#1168)

* feat: Add option to update datatracker references With new commandline option `--use-bib`, xinclude with datatracker.ietf.org are replaced with bib.ietf.org. Fixes #1167 * test: Test DatatrackerToBibConverter * chore: Run yestests (pre-changes) * chore: Run yestests (post-changes)
ietf-tools · Dec 5, 2024 · 924f192 · 924f192
1 parent 2c52c6f
commit 924f192
Show file tree

Hide file tree

Showing 20 changed files with 186 additions and 83 deletions.
diff --git a/test.py b/test.py
@@ -12,6 +12,7 @@
 from xml2rfc.boilerplate_rfc_7841 import boilerplate_rfc_status_of_memo
 from xml2rfc.walkpdf import xmldoc
 from xml2rfc.writers.base import default_options, BaseV3Writer, RfcWriterError
+from xml2rfc.writers import DatatrackerToBibConverter
 from xml2rfc.writers.text import MAX_WIDTH
 
 try:
@@ -847,5 +848,39 @@ def test_validate_draft_name(self):
             self.writer.validate_draft_name()
 
 
+class DatatrackerToBibConverterTest(unittest.TestCase):
+    """DatatrackerToBibConverter tests"""
+
+    def setUp(self):
+        xml2rfc.log.quiet = True
+        path = "tests/input/elements.xml"
+        self.parser = xml2rfc.XmlRfcParser(path,
+                                           quiet=True,
+                                           options=default_options,
+                                           **options_for_xmlrfcparser)
+        self.xmlrfc = self.parser.parse()
+        self.writer = xml2rfc.DatatrackerToBibConverter(self.xmlrfc, quiet=True)
+
+    def test_convert(self):
+        reference_a = "reference.I-D.ietf-sipcore-multiple-reasons.xml"
+        reference_b = "reference.I-D.draft-ietf-sipcore-multiple-reasons-01.xml"
+        rfc = lxml.etree.fromstring(f"""
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude">
+    <xi:include href="https://datatracker.ietf.org/doc/bibxml3/{reference_a}"/>
+    <xi:include href="https://datatracker.ietf.org/doc/bibxml3/{reference_b}"/>
+</rfc>""")
+        self.writer.root = rfc
+        self.writer.convert_xincludes()
+
+        ns = {"xi": b"http://www.w3.org/2001/XInclude"}
+        xincludes = self.writer.root.xpath("//xi:include", namespaces=ns)
+
+        # reference without revision
+        self.assertEqual(xincludes[0].get("href"), f"https://bib.ietf.org/public/rfc/bibxml-ids/{reference_a}")
+
+        # reference with revision
+        self.assertEqual(xincludes[1].get("href"), f"https://bib.ietf.org/public/rfc/bibxml-ids/{reference_b}")
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/valid/docfile.html b/tests/valid/docfile.html
@@ -4,15 +4,15 @@
 <meta charset="utf-8">
 <meta content="Cherokee,Common,Greek,Latin" name="scripts">
 <meta content="initial-scale=1.0" name="viewport">
-<title>Xml2rfc Vocabulary Version 3 Schema xml2rfc release 3.24.0</title>
+<title>Xml2rfc Vocabulary Version 3 Schema xml2rfc release 3.25.0</title>
 <meta content="xml2rfc(1)" name="author">
 <meta content="
        
        This document provides information about the XML schema implemented in this release of xml2rfc, and the individual elements of that schema.  The document is generated from the RNG schema file that is part of the xml2rfc distribution, so schema information in this document should always be in sync with the schema in actual use.  The textual descriptions depend on manual updates in order to reflect the implementation.
        
     " name="description">
-<meta content="xml2rfc 3.24.0" name="generator">
-<meta content="xml2rfc-docs-3.24.0" name="ietf.draft">
+<meta content="xml2rfc 3.25.0" name="generator">
+<meta content="xml2rfc-docs-3.25.0" name="ietf.draft">
 <link href="tests/out/docfile.xml" rel="alternate" type="application/rfc+xml">
 <link href="#copyright" rel="license">
 <link href="xml2rfc.css" rel="stylesheet">
@@ -24,7 +24,7 @@
 <thead><tr>
 <td class="left"></td>
 <td class="center">Xml2rfc Vocabulary V3 Schema</td>
-<td class="right">November 2024</td>
+<td class="right">December 2024</td>
 </tr></thead>
 <tfoot><tr>
 <td class="left">xml2rfc(1)</td>
@@ -39,7 +39,7 @@
 <dd class="workgroup">xml2rfc(1)</dd>
 <dt class="label-published">Published:</dt>
 <dd class="published">
-<time datetime="2024-11-14" class="published">14 November 2024</time>
+<time datetime="2024-12-04" class="published">4 December 2024</time>
     </dd>
 <dt class="label-authors">Author:</dt>
 <dd class="authors">
@@ -49,7 +49,7 @@
 </dd>
 </dl>
 </div>
-<h1 id="title">Xml2rfc Vocabulary Version 3 Schema<br>xml2rfc release 3.24.0</h1>
+<h1 id="title">Xml2rfc Vocabulary Version 3 Schema<br>xml2rfc release 3.25.0</h1>
 <section id="section-abstract">
       <h2 id="abstract"><a href="#abstract" class="selfRef">Abstract</a></h2>
 <p id="section-abstract-1">
@@ -371,7 +371,7 @@ <h2 id="name-introduction">
 <p id="section-1-5">
        The latest version of this documentation is available in HTML form at <span><a href="https://ietf-tools.github.io/xml2rfc/">https://ietf-tools.github.io/xml2rfc/</a></span>.<a href="#section-1-5" class="pilcrow">¶</a></p>
 <p id="section-1-6">
-            This documentation applies to xml2rfc version 3.24.0.<a href="#section-1-6" class="pilcrow">¶</a></p>
+            This documentation applies to xml2rfc version 3.25.0.<a href="#section-1-6" class="pilcrow">¶</a></p>
 </section>
 <section id="section-2">
       <h2 id="name-schema-version-3-elements">
@@ -5616,7 +5616,7 @@ <h3 id="name-format-selection">
         </h3>
 <p id="appendix-B.3-1">
                One or more of the following output formats may be specified. The default is --text. The destination filename will be based on the input filename, unless --out=FILE or --basename=BASE is used.
-               The group has 10 options.<a href="#appendix-B.3-1" class="pilcrow">¶</a></p>
+               The group has 11 options.<a href="#appendix-B.3-1" class="pilcrow">¶</a></p>
 <span class="break"></span><dl class="dlNewline" id="appendix-B.3-2">
           <dt id="appendix-B.3-2.1">
 <div id="option--text">
@@ -5673,39 +5673,48 @@ <h3 id="name-format-selection">
 </dd>
           <dd class="break"></dd>
 <dt id="appendix-B.3-2.13">
-<div id="option--v2v3">
-<code>--v2v3</code>                    </div>
+<div id="option--use-bib">
+<code>--use-bib</code>                    </div>
           </dt>
 <dd style="margin-left: 1.5em" id="appendix-B.3-2.14">
             <p id="appendix-B.3-2.14.1">
-                          Convert vocabulary version 2 XML to version 3.<a href="#appendix-B.3-2.14.1" class="pilcrow">¶</a></p>
+                          Update all datatracker references with bib.ietf.org.<a href="#appendix-B.3-2.14.1" class="pilcrow">¶</a></p>
 </dd>
           <dd class="break"></dd>
 <dt id="appendix-B.3-2.15">
-<div id="option--preptool">
-<code>--preptool</code>                    </div>
+<div id="option--v2v3">
+<code>--v2v3</code>                    </div>
           </dt>
 <dd style="margin-left: 1.5em" id="appendix-B.3-2.16">
             <p id="appendix-B.3-2.16.1">
-                          Run preptool on the input.<a href="#appendix-B.3-2.16.1" class="pilcrow">¶</a></p>
+                          Convert vocabulary version 2 XML to version 3.<a href="#appendix-B.3-2.16.1" class="pilcrow">¶</a></p>
 </dd>
           <dd class="break"></dd>
 <dt id="appendix-B.3-2.17">
-<div id="option--unprep">
-<code>--unprep</code>                    </div>
+<div id="option--preptool">
+<code>--preptool</code>                    </div>
           </dt>
 <dd style="margin-left: 1.5em" id="appendix-B.3-2.18">
             <p id="appendix-B.3-2.18.1">
-                          Reduce prepped xml to unprepped.<a href="#appendix-B.3-2.18.1" class="pilcrow">¶</a></p>
+                          Run preptool on the input.<a href="#appendix-B.3-2.18.1" class="pilcrow">¶</a></p>
 </dd>
           <dd class="break"></dd>
 <dt id="appendix-B.3-2.19">
-<div id="option--info">
-<code>--info</code>                    </div>
+<div id="option--unprep">
+<code>--unprep</code>                    </div>
           </dt>
 <dd style="margin-left: 1.5em" id="appendix-B.3-2.20">
             <p id="appendix-B.3-2.20.1">
-                          Generate a JSON file with anchor to section lookup information.<a href="#appendix-B.3-2.20.1" class="pilcrow">¶</a></p>
+                          Reduce prepped xml to unprepped.<a href="#appendix-B.3-2.20.1" class="pilcrow">¶</a></p>
+</dd>
+          <dd class="break"></dd>
+<dt id="appendix-B.3-2.21">
+<div id="option--info">
+<code>--info</code>                    </div>
+          </dt>
+<dd style="margin-left: 1.5em" id="appendix-B.3-2.22">
+            <p id="appendix-B.3-2.22.1">
+                          Generate a JSON file with anchor to section lookup information.<a href="#appendix-B.3-2.22.1" class="pilcrow">¶</a></p>
 </dd>
         <dd class="break"></dd>
 </dl>
@@ -6387,7 +6396,7 @@ <h2 id="name-xml2rfc-documentation-templ">
 <p id="appendix-D-1">
 
             The following variables are available for use in an xml2rfc
-            manpage Jinja2 template, as of xml2rfc version 3.24.0:<a href="#appendix-D-1" class="pilcrow">¶</a></p>
+            manpage Jinja2 template, as of xml2rfc version 3.25.0:<a href="#appendix-D-1" class="pilcrow">¶</a></p>
 <span class="break"></span><dl class="dlNewline" id="appendix-D-2">
         <dt id="appendix-D-2.1">{{ bare_latin_tags }}:</dt>
         <dd style="margin-left: 1.5em" id="appendix-D-2.2"></dd>

diff --git a/tests/valid/draft-miek-test.html b/tests/valid/draft-miek-test.html
@@ -16,7 +16,7 @@
             This version is adapted to work with "xml2rfc" version 2.x.  
        
     ' name="description">
-<meta content="xml2rfc 3.24.0" name="generator">
+<meta content="xml2rfc 3.25.0" name="generator">
 <meta content="RFC" name="keyword">
 <meta content="Request for Comments" name="keyword">
 <meta content="I-D" name="keyword">
@@ -26,7 +26,7 @@
 <meta content="Extensible Markup Language" name="keyword">
 <meta content="draft-gieben-writing-rfcs-pandoc-02" name="ietf.draft">
 <!-- Generator version information:
-  xml2rfc 3.24.0
+  xml2rfc 3.25.0
     Python 3.12.7
     ConfigArgParse 1.7
     google-i18n-address 3.1.1

diff --git a/tests/valid/draft-template.html b/tests/valid/draft-template.html
@@ -11,11 +11,11 @@
        Insert an abstract: MANDATORY. This template is for creating an
       Internet Draft. 
     " name="description">
-<meta content="xml2rfc 3.24.0" name="generator">
+<meta content="xml2rfc 3.25.0" name="generator">
 <meta content="template" name="keyword">
 <meta content="draft-ietf-xml2rfc-template-05" name="ietf.draft">
 <!-- Generator version information:
-  xml2rfc 3.24.0
+  xml2rfc 3.25.0
     Python 3.12.7
     ConfigArgParse 1.7
     google-i18n-address 3.1.1

diff --git a/tests/valid/indexes.pages.text b/tests/valid/indexes.pages.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                         November 14, 2024
+Internet-Draft                                          December 4, 2024
 Intended status: Experimental                                           
-Expires: May 18, 2025
+Expires: June 7, 2025
 
 
                           xml2rfc index tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on May 18, 2025.
+   This Internet-Draft will expire on June 7, 2025.
 
 Copyright Notice
 
@@ -53,9 +53,9 @@ Table of Contents
 
 
 
-Person                    Expires May 18, 2025                  [Page 1]
+Person                    Expires June 7, 2025                  [Page 1]
 
-Internet-Draft             xml2rfc index tests             November 2024
+Internet-Draft             xml2rfc index tests             December 2024
 
 
    This is another section!
@@ -109,9 +109,9 @@ Index
 
 
 
-Person                    Expires May 18, 2025                  [Page 2]
+Person                    Expires June 7, 2025                  [Page 2]
 
-Internet-Draft             xml2rfc index tests             November 2024
+Internet-Draft             xml2rfc index tests             December 2024
 
 
       E
@@ -165,4 +165,4 @@ Author's Address
 
 
 
-Person                    Expires May 18, 2025                  [Page 3]
+Person                    Expires June 7, 2025                  [Page 3]
diff --git a/tests/valid/indexes.prepped.xml b/tests/valid/indexes.prepped.xml
@@ -1,6 +1,6 @@
 <?xml version='1.0' encoding='utf-8'?>
-<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" sortRefs="true" category="exp" submissionType="independent" ipr="trust200902" docName="indexes-00" indexInclude="true" prepTime="2024-11-14T10:46:57" scripts="Common,Latin" symRefs="true" tocDepth="3" tocInclude="true">
-  <!-- xml2rfc v2v3 conversion 3.24.0 -->
+<rfc xmlns:xi="http://www.w3.org/2001/XInclude" version="3" sortRefs="true" category="exp" submissionType="independent" ipr="trust200902" docName="indexes-00" indexInclude="true" prepTime="2024-12-04T00:11:42" scripts="Common,Latin" symRefs="true" tocDepth="3" tocInclude="true">
+  <!-- xml2rfc v2v3 conversion 3.25.0 -->
 
 
 
@@ -20,7 +20,7 @@
         </postal>
       </address>
     </author>
-    <date day="14" month="11" year="2024"/>
+    <date day="04" month="12" year="2024"/>
     <boilerplate>
       <section anchor="status-of-memo" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.1">
         <name slugifiedName="name-status-of-this-memo">Status of This Memo</name>
@@ -41,7 +41,7 @@
         material or to cite them other than as "work in progress."
         </t>
         <t indent="0" pn="section-boilerplate.1-4">
-        This Internet-Draft will expire on 18 May 2025.
+        This Internet-Draft will expire on 7 June 2025.
         </t>
       </section>
       <section anchor="copyright" numbered="false" removeInRFC="false" toc="exclude" pn="section-boilerplate.2">

diff --git a/tests/valid/indexes.text b/tests/valid/indexes.text
@@ -3,9 +3,9 @@
 
 
 Network Working Group                                     H. Person, Ed.
-Internet-Draft                                         November 14, 2024
+Internet-Draft                                          December 4, 2024
 Intended status: Experimental                                           
-Expires: May 18, 2025
+Expires: June 7, 2025
 
 
                           xml2rfc index tests
@@ -26,7 +26,7 @@ Status of This Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on May 18, 2025.
+   This Internet-Draft will expire on June 7, 2025.
 
 Copyright Notice
 

diff --git a/tests/valid/indexes.v3.html b/tests/valid/indexes.v3.html
@@ -6,7 +6,7 @@
 <meta content="initial-scale=1.0" name="viewport">
 <title>xml2rfc index tests</title>
 <meta content="Human Person" name="author">
-<meta content="xml2rfc 3.24.0" name="generator">
+<meta content="xml2rfc 3.25.0" name="generator">
 <meta content="indexes-00" name="ietf.draft">
 <link href="tests/input/indexes.xml" rel="alternate" type="application/rfc+xml">
 <link href="#copyright" rel="license">
@@ -19,11 +19,11 @@
 <thead><tr>
 <td class="left">Internet-Draft</td>
 <td class="center">xml2rfc index tests</td>
-<td class="right">November 2024</td>
+<td class="right">December 2024</td>
 </tr></thead>
 <tfoot><tr>
 <td class="left">Person</td>
-<td class="center">Expires May 18, 2025</td>
+<td class="center">Expires June 7, 2025</td>
 <td class="right">[Page]</td>
 </tr></tfoot>
 </table>
@@ -36,12 +36,12 @@
 <dd class="internet-draft">indexes-00</dd>
 <dt class="label-published">Published:</dt>
 <dd class="published">
-<time datetime="2024-11-14" class="published">November 14, 2024</time>
+<time datetime="2024-12-04" class="published">December 4, 2024</time>
     </dd>
 <dt class="label-intended-status">Intended Status:</dt>
 <dd class="intended-status">Experimental</dd>
 <dt class="label-expires">Expires:</dt>
-<dd class="expires"><time datetime="2025-05-18">May 18, 2025</time></dd>
+<dd class="expires"><time datetime="2025-06-07">June 7, 2025</time></dd>
 <dt class="label-authors">Author:</dt>
 <dd class="authors">
 <div class="author">
@@ -71,7 +71,7 @@ <h2 id="name-status-of-this-memo">
         time. It is inappropriate to use Internet-Drafts as reference
         material or to cite them other than as "work in progress."<a href="#section-boilerplate.1-3" class="pilcrow">¶</a></p>
 <p id="section-boilerplate.1-4">
-        This Internet-Draft will expire on May 18, 2025.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
+        This Internet-Draft will expire on June 7, 2025.<a href="#section-boilerplate.1-4" class="pilcrow">¶</a></p>
 </section>
 </div>
 <div id="copyright">