From d2bbdf05d0f3f0e04a5201b3a0db3af2177a6dfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Benqu=C3=A9?= <mail@davidbenque.com>
Date: Mon, 9 Aug 2021 11:56:34 +0100
Subject: [PATCH] Add sitemap and robots.txt

---
 conf.py        |  6 +++++-
 poetry.lock    | 29 ++++++++++++++++++++++++++++-
 pyproject.toml |  1 +
 robots.txt     |  4 ++++
 4 files changed, 38 insertions(+), 2 deletions(-)
 create mode 100644 robots.txt

diff --git a/conf.py b/conf.py
index c985a983..9c776d18 100644
--- a/conf.py
+++ b/conf.py
@@ -41,7 +41,8 @@
 extensions = [
     'recommonmark',
     'sphinx_markdown_tables',
-    'sphinx_fontawesome'
+    'sphinx_fontawesome',
+    'sphinx_sitemap'
 ]
 
 source_suffix = {
@@ -111,6 +112,9 @@
 using_rtd_theme = True
 smartquotes = False
 
+html_baseurl = 'https://docs.cryptpad.fr/' # for the generated site-map
+html_extra_path = ["robots.txt"]
+
 html_theme_options = {
     'logo_only': False,
     'display_version': True,
diff --git a/poetry.lock b/poetry.lock
index e73762db..fb2d5abc 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -183,6 +183,14 @@ urllib3 = ">=1.21.1,<1.27"
 security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
 socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
 
+[[package]]
+name = "six"
+version = "1.16.0"
+description = "Python 2 and 3 compatibility utilities"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+
 [[package]]
 name = "snowballstemmer"
 version = "2.1.0"
@@ -261,6 +269,18 @@ python-versions = "*"
 [package.dependencies]
 markdown = ">=3.0.1"
 
+[[package]]
+name = "sphinx-sitemap"
+version = "2.2.0"
+description = "Sitemap generator for Sphinx"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+six = "*"
+sphinx = ">=1.2"
+
 [[package]]
 name = "sphinxcontrib-applehelp"
 version = "1.0.2"
@@ -348,7 +368,7 @@ brotli = ["brotlipy (>=0.6.0)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "ffc7f8a401c30cf3e16e53ab65b1a1caf5533d9a11c63c7a2cb559decbd23261"
+content-hash = "d7bebb240812c4c2f9c5dd75a98cfe47c282be29743ef2c1d556f78aa22ce096"
 
 [metadata.files]
 alabaster = [
@@ -477,6 +497,10 @@ requests = [
     {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
     {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
 ]
+six = [
+    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
+    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
+]
 snowballstemmer = [
     {file = "snowballstemmer-2.1.0-py2.py3-none-any.whl", hash = "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2"},
     {file = "snowballstemmer-2.1.0.tar.gz", hash = "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"},
@@ -496,6 +520,9 @@ sphinx-markdown-tables = [
     {file = "sphinx-markdown-tables-0.0.15.tar.gz", hash = "sha256:722b70a3a14156b6777cf04920d015215eb65b9dfde5cc72f430763ff5e84173"},
     {file = "sphinx_markdown_tables-0.0.15-py3-none-any.whl", hash = "sha256:24a37662d86ee8bceb7d4f7003df0b25bc52401369d1ddc40d13ae7b58697031"},
 ]
+sphinx-sitemap = [
+    {file = "sphinx-sitemap-2.2.0.tar.gz", hash = "sha256:65adda39233cb17c0da10ba1cebaa2df73e271cdb6f8efd5cec8eef3b3cf7737"},
+]
 sphinxcontrib-applehelp = [
     {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"},
     {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"},
diff --git a/pyproject.toml b/pyproject.toml
index f8301147..210df22b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ recommonmark = "^0.7.1"
 sphinx-intl = "^2.0.1"
 sphinx-markdown-tables = "^0.0.15"
 sphinx_fontawesome = "^0.0.6"
+sphinx-sitemap = "^2.2.0"
 
 [tool.poetry.dev-dependencies]
 
diff --git a/robots.txt b/robots.txt
new file mode 100644
index 00000000..28198f97
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://docs.cryptpad.fr/en/sitemap.xml
\ No newline at end of file