diff --git a/apidocs/crawlerConfig.json b/apidocs/crawlerConfig.json index a2b20f8..5ae1fc6 100644 --- a/apidocs/crawlerConfig.json +++ b/apidocs/crawlerConfig.json @@ -1,38 +1,33 @@ { "index_name": "copper3d-visualisation", "start_urls": ["https://linkungao.github.io/copper3d_visualisation/"], - "sitemap_urls": [ - "https://linkungao.github.io/copper3d_visualisation/sitemap.xml" - ], - "sitemap_alternate_links": true, - "stop_urls": ["/tests"], + "rateLimit": 8, + "maxDepth": 10, "selectors": { "lvl0": { - "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class, 'navbar__link--active')]/text())[last()]", - "type": "xpath", - "global": true, - "default_value": "Documentation" + "selector": "", + "defaultValue": "Documentation" }, "lvl1": ".content h1", "lvl2": ".content h2", "lvl3": ".content h3", "lvl4": ".content h4", "lvl5": ".content h5", - "content": ".content p, .content li" + "content": ".content p, .content li", + "lang": { + "selector": "/html/@lang", + "type": "xpath", + "global": true + } }, - "strip_chars": " .,;:#", + "selectors_exclude": [ + "aside", + ".page-footer", + ".next-and-prev-link", + ".table-of-contents" + ], "custom_settings": { - "separatorsToIndex": "_", - "attributesForFaceting": ["language", "version", "type", "docusaurus_tag"], - "attributesToRetrieve": [ - "hierarchy", - "content", - "anchor", - "url", - "url_without_anchor", - "type" - ] + "attributesForFaceting": ["lang", "tags"] }, - "conversation_id": ["833762294"], - "nb_hits": 46250 + "js_render": true } diff --git a/apidocs/crawlerConfig_1.json b/apidocs/crawlerConfig_1.json deleted file mode 100644 index e2f7c6d..0000000 --- a/apidocs/crawlerConfig_1.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "index_name": "copper3d-visualisation", - "start_urls": ["/copper3d_visualisation"], - "rateLimit": 8, - "maxDepth": 10, - "selectors": { - "lvl0": { - "selector": "", - "defaultValue": "Documentation" - }, - "lvl1": ".content h1", - "lvl2": ".content h2", - "lvl3": ".content h3", - "lvl4": ".content h4", - "lvl5": ".content h5", - "content": ".content p, .content li", - "lang": { - "selector": "/html/@lang", - "type": "xpath", - "global": true - } - }, - "selectors_exclude": [ - "aside", - ".page-footer", - ".next-and-prev-link", - ".table-of-contents" - ], - "custom_settings": { - "attributesForFaceting": ["lang", "tags"] - }, - "js_render": true -}