Add create_index function for building documentation indexes

AnswerDotAI · Dec 2, 2024 · 9b982f1 · 9b982f1
1 parent aec2cfc
commit 9b982f1
Show file tree

Hide file tree

Showing 3 changed files with 204 additions and 6 deletions.
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
@@ -70,6 +70,7 @@
                                 'nbdev.doclinks._qual_sym': ('api/doclinks.html#_qual_sym', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._qual_syms': ('api/doclinks.html#_qual_syms', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._sym_nm': ('api/doclinks.html#_sym_nm', 'nbdev/doclinks.py'),
+                                'nbdev.doclinks.create_index': ('api/doclinks.html#create_index', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks.nbdev_export': ('api/doclinks.html#nbdev_export', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks.nbglob': ('api/doclinks.html#nbglob', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks.nbglob_cli': ('api/doclinks.html#nbglob_cli', 'nbdev/doclinks.py'),

diff --git a/nbdev/doclinks.py b/nbdev/doclinks.py
@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/05_doclinks.ipynb.
 
 # %% auto 0
-__all__ = ['patch_name', 'nbglob', 'nbglob_cli', 'nbdev_export', 'NbdevLookup']
+__all__ = ['typs', 'bset', 'patch_name', 'nbglob', 'nbglob_cli', 'nbdev_export', 'create_index', 'NbdevLookup']
 
 # %% ../nbs/api/05_doclinks.ipynb
 from .config import *
@@ -14,11 +14,14 @@
 from fastcore.script import *
 from fastcore.utils import *
 from fastcore.meta import delegates
+from fastcore.net import urlread
 
-import ast,contextlib
+import ast,builtins,contextlib
 import pkg_resources,importlib
-from astunparse import unparse
 
+from astunparse import unparse
+from io import BytesIO
+from collections import defaultdict
 from pprint import pformat
 from urllib.parse import urljoin
 from functools import lru_cache
@@ -153,6 +156,28 @@ def nbdev_export(
     add_init(get_config().lib_path)
     _build_modidx()
 
+# %% ../nbs/api/05_doclinks.ipynb
+typs = 'module','class','method','function'
+bset = set(dir(builtins))
+
+# %% ../nbs/api/05_doclinks.ipynb
+def create_index(url, pre=None):
+    "Create a documentation index from a sphinx inventory file at `url`, with optional prefix `pre`"
+    try: from sphinx.util.inventory import InventoryFile
+    except ImportError: print('`sphinx` is a dependency for building indexes. Run `pip install sphinx`.')
+    pre = ifnone(pre, f"{url}/")
+    invs = urlread(f'{url}/objects.inv', decode=False)
+    idx = InventoryFile.load(stream=BytesIO(invs), uri=pre, joinfunc=urljoin)
+    _get = lambda o: {k:v[2] for k,v in idx[f'py:{o}'].items() if k[0]!='_'}
+    d = {o:_get(o) for o in typs}
+    syms = defaultdict(dict)
+    for o in typs:
+        for k,v in d[o].items():
+            if k.split('.')[0] in bset: k = 'builtins.' + k
+            modparts = k.split(".")[:-2 if o=='method' else -1]
+            if modparts: syms['.'.join(modparts)][k] = v
+    return syms
+
 # %% ../nbs/api/05_doclinks.ipynb
 import importlib,ast
 from functools import lru_cache

diff --git a/nbs/api/05_doclinks.ipynb b/nbs/api/05_doclinks.ipynb
@@ -33,11 +33,14 @@
     "from fastcore.script import *\n",
     "from fastcore.utils import *\n",
     "from fastcore.meta import delegates\n",
+    "from fastcore.net import urlread\n",
     "\n",
-    "import ast,contextlib\n",
+    "import ast,builtins,contextlib\n",
     "import pkg_resources,importlib\n",
-    "from astunparse import unparse\n",
     "\n",
+    "from astunparse import unparse\n",
+    "from io import BytesIO\n",
+    "from collections import defaultdict\n",
     "from pprint import pformat\n",
     "from urllib.parse import urljoin\n",
     "from functools import lru_cache\n",
@@ -355,7 +358,6 @@
    "outputs": [],
    "source": [
     "#|export\n",
-    "\n",
     "@call_parse\n",
     "@delegates(nbglob_cli)\n",
     "def nbdev_export(\n",
@@ -383,6 +385,88 @@
     "N.B.: the `black_format` processor is passed in by default. But it is a no-op, unless `black_formatting=True` is set in your `settings.ini` configuration. You can omit it from `nbdev_export` on the command line by passing in `--procs`."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Construct Index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "typs = 'module','class','method','function'\n",
+    "bset = set(dir(builtins))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def create_index(url, pre=None):\n",
+    "    \"Create a documentation index from a sphinx inventory file at `url`, with optional prefix `pre`\"\n",
+    "    try: from sphinx.util.inventory import InventoryFile\n",
+    "    except ImportError: print('`sphinx` is a dependency for building indexes. Run `pip install sphinx`.')\n",
+    "    pre = ifnone(pre, f\"{url}/\")\n",
+    "    invs = urlread(f'{url}/objects.inv', decode=False)\n",
+    "    idx = InventoryFile.load(stream=BytesIO(invs), uri=pre, joinfunc=urljoin)\n",
+    "    _get = lambda o: {k:v[2] for k,v in idx[f'py:{o}'].items() if k[0]!='_'}\n",
+    "    d = {o:_get(o) for o in typs}\n",
+    "    syms = defaultdict(dict)\n",
+    "    for o in typs:\n",
+    "        for k,v in d[o].items():\n",
+    "            if k.split('.')[0] in bset: k = 'builtins.' + k\n",
+    "            modparts = k.split(\".\")[:-2 if o=='method' else -1]\n",
+    "            if modparts: syms['.'.join(modparts)][k] = v\n",
+    "    return syms"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = 'https://docs.python.org/3'\n",
+    "syms = create_index(url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'builtins.bool': 'https://docs.python.org/3/library/functions.html#bool',\n",
+       " 'builtins.bytearray': 'https://docs.python.org/3/library/stdtypes.html#bytearray',\n",
+       " 'builtins.bytes': 'https://docs.python.org/3/library/stdtypes.html#bytes',\n",
+       " 'builtins.complex': 'https://docs.python.org/3/library/functions.html#complex',\n",
+       " 'builtins.dict': 'https://docs.python.org/3/library/stdtypes.html#dict',\n",
+       " 'builtins.float': 'https://docs.python.org/3/library/functions.html#float',\n",
+       " 'builtins.frozenset': 'https://docs.python.org/3/library/stdtypes.html#frozenset',\n",
+       " 'builtins.int': 'https://docs.python.org/3/library/functions.html#int',\n",
+       " 'builtins.list': 'https://docs.python.org/3/library/stdtypes.html#list',\n",
+       " 'builtins.memoryview': 'https://docs.python.org/3/library/stdtypes.html#memoryview'}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dict(list(syms['builtins'].items())[:10])"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -962,6 +1046,94 @@
     "assert NbdevLookup().linkify(md) == md"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'[`abc.ABC`](https://docs.python.org/3/library/abc.html#abc.ABC)'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Test code blocks\n",
+    "md = \"`abc.ABC`\"\n",
+    "NbdevLookup().linkify(md)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'[`functools.partial`](https://docs.python.org/3/library/functools.html#functools.partial)'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Test code blocks\n",
+    "md = \"`functools.partial`\"\n",
+    "NbdevLookup().linkify(md)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'`str.split`'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Test code blocks\n",
+    "md = \"`str.split`\"\n",
+    "NbdevLookup().linkify(md)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'[`numpy.array`](https://numpy.org/doc/stable/reference/generated/numpy.array.html#numpy.array)'"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Test code blocks\n",
+    "md = \"`numpy.array`\"\n",
+    "NbdevLookup().linkify(md)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},