feat(core): add support of remote markdown files

This feature allows you to embed raw content of
markdown files from GitHub and BitBucket repositories.
At this moment, only public repositories are supported.

README.md

@@ -10,6 +10,7 @@ Antmarky is a static-site generator for Markdown based on Node.js/EJS.
 * Fully responsive layout
 * Fully static (doesn't require web server to work)
 * No language frameworks included
+* [Include remote Markdown files from GitHub and BitBucket][remote-md-files]
 * [Markdown][markdown] flavor: `GitHub`. Supported syntax:
     * Heading ids
     * Emojis :tada:
@@ -24,6 +25,7 @@ Antmarky is a static-site generator for Markdown based on Node.js/EJS.
 * [FontAwesome][fa]
 * [Task lists][tasks-list]
 
+[remote-md-files]: features.md#remote-markdown-files
 [markdown]: markdown.md
 [admonitions]: features.md#admonitions
 [syntax-highlight]: features.md#syntax-highlighting

docs/features.md

@@ -1,5 +1,28 @@
 # Features
 
+## Remote markdown files
+
+You can include remote Markdown files in **raw** format from **GitHub** and **BitBucket** public repositories using `!!+` directive:
+
+```
+!!+ github.com/link/to/your/raw/markdown/file.md
+!!+ bitbucket.org/link/to/your/raw/markdown/file.md
+```
+
+::: tip "What is raw format"
+**GitHub raw format**
+
+```
+https://github.com/<username>/<repo>/raw/<branch>/filename.md
+```
+
+**BitBucket raw format**
+
+```
+https://bitbucket.org/<username>/<repo>/raw/<branch>/filename.md
+```
+:::
+
 ## Admonitions
 
 **info**

package.json

@@ -11,6 +11,7 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "axios": "^0.25.0",
     "ejs": "^3.1.6",
     "express": "^4.17.1",
     "fs-extra": "^10.0.0",

server.js

@@ -4,9 +4,7 @@ const path = require('path');
 const app = express();
 const PORT = 8000;
 const { serveContent } = require('./src/commands/serve');
-const { convertCrossLinks } = require('./src/common/prepare-content');
 let { md } = require('./src/common/md-parser');
-const { buildToc } = require('./src/toc');
 let { errorPage } = require('./src/data/defaults');
 app.use(serveContent);
 
@@ -19,7 +17,7 @@ app.get('/', (req, res) => {
   let specificPageData = res.locals.files_data.find(page => page.name == 'README');
   renderData.name = (specificPageData) ? specificPageData.name : '/';
   renderData.title = (specificPageData) ? specificPageData.title : 'Home';
-  renderData.content = (specificPageData) ? specificPageData.html : md.makeHtml(convertCrossLinks(fs.readFileSync(path.resolve('README.md'), 'utf-8')));
+  renderData.content = (specificPageData) ? specificPageData.html : md.makeHtml(fs.readFileSync(path.resolve('README.md'), 'utf-8'));
   renderData.pages = res.locals.all_pages.filter(page => page.name !== 'README');
   res.render('index', renderData);
 });
@@ -33,7 +31,7 @@ app.get('/:pageName.html', (req, res) => {
       name: specificPageData.name,
       title: pageTitle,
       content,
-      toc: buildToc(content),
+      toc: specificPageData.toc,
       pages: res.locals.all_pages.filter(page => page.name !== 'README')
     })
   } else {

src/assets/css/styles.css

@@ -143,12 +143,6 @@ blockquote p {
   margin: 0 5px;
 }
 
-img {
-  display: block;
-  width: 50%;
-  margin: 0 auto;
-}
-
 .callout {
   padding: .5rem 1.25rem;
   margin-top: 1.25rem;

src/commands/serve.js

@@ -1,10 +1,12 @@
-let { findMdFiles, getFilesContent, convertMdToHtml } = require('../common/prepare-content')
+let { findMdFiles, getFilesContent, convertMdToHtml } = require('../common/prepare-content');
+let { embedRemoteMarkdown } = require('../common/embed-remote-markdown');
 
 let serveContent = async (req, res, next) => {
   let locatedMdFiles = await findMdFiles();
   let allPages = locatedMdFiles;
-  let filesMdContent = await getFilesContent(locatedMdFiles);
-  let htmlContent = convertMdToHtml(filesMdContent);
+  let mdFilesContent = await getFilesContent(locatedMdFiles);
+  let mdFilesWithRemoteContent = await embedRemoteMarkdown(mdFilesContent);
+  let htmlContent = convertMdToHtml(mdFilesWithRemoteContent);
   res.locals.files_data = htmlContent;
   res.locals.all_pages = allPages;
   next();

src/common/embed-remote-markdown.js

@@ -0,0 +1,35 @@
+const axios = require('axios');
+
+let fileInclusion = async (url) => {
+  try {
+    let response = await axios.get(url);
+    return response.data;
+  } catch (error) {
+    console.log(error);
+  }
+}
+
+let embedRemoteMarkdown = async (mdFilesContent) => {
+  let remoteContentExtractionRegex = /!!\+ (?<url>https:\/\/(?:github.com|bitbucket.org)\/[\S\s]*?.md)/g;
+  return await Promise.all(mdFilesContent.map(async file => {
+    let resultingContent = file.content;
+    let isRemoteContent = file.content.match(remoteContentExtractionRegex);
+    if (isRemoteContent) {
+      let matches = file.content.matchAll(remoteContentExtractionRegex);
+      for (let match of matches) {
+        resultingContent = await fileInclusion(match.groups.url).then(content => {
+          return resultingContent.replace(match[0], content);
+        });
+      }
+    }
+    return {
+      name: file.name,
+      title: file.title,
+      content: resultingContent
+    }
+  }));
+}
+
+module.exports = {
+  embedRemoteMarkdown
+}

src/common/prepare-content.js

@@ -4,6 +4,7 @@ const fse = require('fs-extra');
 const fsp = require('fs/promises');
 let { md } = require('./md-parser');
 const path = require('path');
+let { embedRemoteMarkdown } = require('./embed-remote-markdown');
 let { buildToc } = require('../toc');
 let { errorPage } = require('../data/defaults');
 
@@ -36,15 +37,10 @@ let getFilesContent = async (fileDetails) => {
   })
 }
 
-let convertCrossLinks = (mdText) => {
-  return mdText.replace(/\.md/g, '.html')
-}
-
 // CONVERT MARKDOWN FILES TO HTML
 let convertMdToHtml = (mdTextArray) => {  
   return mdTextArray.map(mdText => {
-    let linksInContent = convertCrossLinks(mdText.content);
-    let html = md.makeHtml(linksInContent);
+    let html = md.makeHtml(mdText.content);
     let tableOfCOntents = buildToc(html);
     return {
       name: mdText.name,
@@ -65,8 +61,9 @@ let saveHtmlContent = (filename, htmlContent) => {
 let buildContent = async (docsDir) => {
   let locatedMdFiles = await findMdFiles(docsDir);
   let allPages = locatedMdFiles;
-  let filesMdContent = await getFilesContent(locatedMdFiles);
-  let htmlContent = convertMdToHtml(filesMdContent);
+  let mdFilesContent = await getFilesContent(locatedMdFiles);
+  let mdFilesWithRemoteContent = await embedRemoteMarkdown(mdFilesContent);
+  let htmlContent = convertMdToHtml(mdFilesWithRemoteContent);
   return {
     allPages: allPages,
     htmlContent: htmlContent
@@ -126,7 +123,7 @@ let buildStaticFiles = async (docsDir) => {
           let readyHtml = compiledTemplate({
             name: '/',
             title: 'Home',
-            content: md.makeHtml(convertCrossLinks(fs.readFileSync(path.resolve('README.md'), 'utf-8'))),
+            content: md.makeHtml(fs.readFileSync(path.resolve('README.md'), 'utf-8')),
             pages: sidebarListOfPages
           });
           saveHtmlContent('index.html', readyHtml);
@@ -151,7 +148,6 @@ let buildStaticFiles = async (docsDir) => {
 module.exports = {
   findMdFiles: findMdFiles,
   getFilesContent: getFilesContent,
-  convertCrossLinks: convertCrossLinks,
   convertMdToHtml: convertMdToHtml,
   buildStaticFiles: buildStaticFiles
 }

src/extensions/text-modifications.js

@@ -1,5 +1,11 @@
 let markyText = (text) => {
   return [
+    // CROSS-FILES LINKS md -> html
+    {
+      type: 'output',
+      regex: /<a href="([\s\S]*?).md(#[\s\S]*?)?">([\s\S]*?)<\/a>/g,
+      replace: '<a href="$1.html$2">$3</a>'
+    },
     // ADMONITIONS
     {
       type: 'output',
@@ -29,7 +35,7 @@ let markyText = (text) => {
       type: 'output',
       regex: /==(.*)==/g,
       replace: '<mark>$1</mark>'
-    },
+    }
   ]
 }