Automatically convert GitHub-based projects into MarkBind websites (#698

)

GitHub projects commonly have wikis or docs folders for project
documentation. As MarkBind is especially optimized as a project
documentation tool, we would want to make it easier for projects to
adopt it in their workflow.

Let's add the ability to automatically convert a GitHub wiki or docs
folder into a MarkBind website.

docs/userGuide/cliCommands.md

@@ -76,10 +76,13 @@ MarkBind Command Line Interface (CLI) can be run in the following ways:
 * `<root>`<br>
   Root directory. Default is the current directory.<br>
   {{ icon_example }} `./myWebsite`
+* `-c`, `--convert`<br>
+  Convert an existing GitHub wiki or `docs` folder into a MarkBind website. See [Converting an existing Github project]({{ baseUrl }}/userGuide/markBindInTheProjectWorkflow.html#converting-an-existing-github-project) for more information.
 
 {{ icon_examples }}
 * `markbind init` : Initializes the site in the current working directory.
 * `markbind init ./myWebsite` : Initializes the site in `./myWebsite` directory.
+* `markbind init --convert` : Converts the Github wiki or `docs` folder in the current working directory into a MarkBind website.
 
 <hr><!-- ========================================================================== -->
 

docs/userGuide/markBindInTheProjectWorkflow.md

@@ -36,5 +36,29 @@ You can keep the user docs in a separate directory (say `user-docs`) and set up
 Similarly, you can keep the dev docs in a separate directory (sey `dev-docs`) and set up Netlify to deploy the site when there is an update to the `master` branch; that way, developers can see the latest version of dev-docs via the Netlify site.
 </div>
 
+#### Converting existing project documentation/wiki
+
+MarkBind supports the automatic conversion of an existing GitHub wiki or `docs` folder containing Markdown files.
+
+A MarkBind conversion involves the following:
+- Adding a Home page: If your project already has a `README.md` or `Home.md`, the content will be copied over to `index.md`. Otherwise, a default home page will be added.
+- Adding an About Us page: If your project already has `about.md`, this will be used as the About page. Otherwise, a default About page will be added.
+- Adding a top navigation bar.
+- Adding a site navigation menu: If your project has a valid `_Sidebar.md` file, it will be used as the [site navigation menu](https://markbind.org/userGuide/tweakingThePageStructure.html#site-navigation-menus). Otherwise, the menu will be built from your project's directory structure and contain links to all addressable pages.   
+- Adding a custom footer: If your project has a valid `_Footer.md` file, it will be used as the website footer. Otherwise, a default footer will be added.
+
+<box type="warning">
+    Conversion might not work if your project files have existing Nunjucks syntax. 
+</box>
+
+To convert your existing project, follow these steps:
+1. Navigate into the project directory.
+1. Run `markbind init --convert` to convert the project.
+1. You can now preview the website using `markbind serve` to view your newly converted MarkBind website.
+
+<box type="info">
+    You only need to run the conversion once. Once you have converted your project, you can proceed to edit it as a normal MarkBind project.
+</box> 
+
 {% from "njk/common.njk" import previous_next %}
 {{ previous_next('deployingTheSite', 'themes') }}

index.js

@@ -3,6 +3,7 @@
 
 // Entry file for Markbind project
 const chokidar = require('chokidar');
+const fs = require('fs-extra-promise');
 const liveServer = require('live-server');
 const path = require('path');
 const program = require('commander');
@@ -45,15 +46,33 @@ program
 
 program
   .command('init [root]')
+  .option('-c, --convert', 'convert a GitHub wiki or docs folder to a MarkBind website')
   .alias('i')
   .description('init a markbind website project')
-  .action((root) => {
+  .action((root, options) => {
     const rootFolder = path.resolve(root || process.cwd());
+    const outputRoot = path.join(rootFolder, '_site');
     printHeader();
+    if (options.convert) {
+      if (fs.existsSync(path.resolve(rootFolder, 'site.json'))) {
+        logger.error('Cannot convert an existing MarkBind website!');
+        return;
+      }
+    }
     Site.initSite(rootFolder)
       .then(() => {
         logger.info('Initialization success.');
       })
+      .then(() => {
+        if (options.convert) {
+          logger.info('Converting to MarkBind website.');
+          new Site(rootFolder, outputRoot).convert()
+            .then(() => {
+              logger.info('Conversion success.');
+            })
+            .catch(handleError);
+        }
+      })
       .catch(handleError);
   });
 

src/Site.js

@@ -18,13 +18,15 @@ _.isBoolean = require('lodash/isBoolean');
 _.isUndefined = require('lodash/isUndefined');
 _.noop = require('lodash/noop');
 _.omitBy = require('lodash/omitBy');
+_.startCase = require('lodash/startCase');
 _.union = require('lodash/union');
 _.uniq = require('lodash/uniq');
 
 const url = {};
 url.join = path.posix.join;
 
 const delay = require('./util/delay');
+const FsUtil = require('./util/fsUtil');
 const logger = require('./util/logger');
 const Page = require('./Page');
 
@@ -38,6 +40,7 @@ const TEMP_FOLDER_NAME = '.temp';
 const TEMPLATE_ROOT_FOLDER_NAME = 'template';
 const TEMPLATE_SITE_ASSET_FOLDER_NAME = 'markbind';
 
+const ABOUT_MARKDOWN_FILE = 'about.md';
 const BUILT_IN_PLUGIN_FOLDER_NAME = 'plugins';
 const BUILT_IN_DEFAULT_PLUGIN_FOLDER_NAME = 'plugins/default';
 const FAVICON_DEFAULT_PATH = 'favicon.ico';
@@ -59,6 +62,8 @@ const LAYOUT_FOLDER_PATH = '_markbind/layouts';
 const LAYOUT_SCRIPTS_PATH = 'scripts.js';
 const LAYOUT_SITE_FOLDER_NAME = 'layouts';
 const USER_VARIABLES_PATH = '_markbind/variables.md';
+const WIKI_SITE_NAV_PATH = '_Sidebar.md';
+const WIKI_FOOTER_PATH = '_Footer.md';
 
 function getBootswatchThemePath(theme) {
   return path.join(__dirname, '..', 'node_modules', 'bootswatch', 'dist', theme, 'bootstrap.min.css');
@@ -114,6 +119,9 @@ const SITE_CONFIG_DEFAULT = {
   },
 };
 
+const ABOUT_MARKDOWN_DEFAULT = '# About\n'
+  + 'Welcome to your **About Us** page.\n';
+
 const FOOTER_DEFAULT = '<footer>\n'
   + '  <div class="text-center">\n'
   + '    This is a dynamic height footer that supports markdown <md>:smile:</md>!\n'
@@ -146,6 +154,19 @@ const SITE_NAV_DEFAULT = '<navigation>\n'
   + '* [Home :glyphicon-home:]({{baseUrl}}/index.html)\n'
   + '</navigation>\n';
 
+const TOP_NAV_DEFAULT = '<header><navbar placement="top" type="inverse">\n'
+  + '  <a slot="brand" href="{{baseUrl}}/index.html" title="Home" class="navbar-brand">'
+  + '<i class="far fa-file-image"></i></a>\n'
+  + '  <li><a href="{{baseUrl}}/index.html" class="nav-link">HOME</a></li>\n'
+  + '  <li><a href="{{baseUrl}}/about.html" class="nav-link">ABOUT</a></li>\n'
+  + '  <li slot="right">\n'
+  + '    <form class="navbar-form">\n'
+  + '      <searchbar :data="searchData" placeholder="Search" :on-hit="searchCallback"'
+  + ' menu-align-right></searchbar>\n'
+  + '    </form>\n'
+  + '  </li>\n'
+  + '</navbar></header>';
+
 const LAYOUT_SCRIPTS_DEFAULT = 'MarkBind.afterSetup(() => {\n'
   + '  // Include code to be called after MarkBind setup here.\n'
   + '});\n';
@@ -435,6 +456,142 @@ Site.prototype.createPage = function (config) {
   });
 };
 
+/**
+ * Converts an existing Github wiki or docs folder to a MarkBind website.
+ */
+Site.prototype.convert = function () {
+  return this.readSiteConfig()
+    .then(() => this.collectAddressablePages())
+    .then(() => this.addIndexPage())
+    .then(() => this.addAboutPage())
+    .then(() => this.addTopNavToDefaultLayout())
+    .then(() => this.addFooterToDefaultLayout())
+    .then(() => this.addSiteNavToDefaultLayout())
+    .then(() => this.addDefaultLayoutToSiteConfig())
+    .then(() => this.printBaseUrlMessage());
+};
+
+/**
+ * Copies over README.md or Home.md to default index.md if present.
+ */
+Site.prototype.addIndexPage = function () {
+  const indexPagePath = path.join(this.rootPath, INDEX_MARKDOWN_FILE);
+  const fileNames = ['README.md', 'Home.md'];
+  const filePath = fileNames.find(fileName => fs.existsSync(path.join(this.rootPath, fileName)));
+  // if none of the files exist, do nothing
+  if (_.isUndefined(filePath)) return Promise.resolve();
+  return fs.copyAsync(path.join(this.rootPath, filePath), indexPagePath)
+    .catch(() => Promise.reject(new Error(`Failed to copy over ${filePath}`)));
+};
+
+/**
+ * Adds an about page to site if not present.
+ */
+Site.prototype.addAboutPage = function () {
+  const aboutPath = path.join(this.rootPath, ABOUT_MARKDOWN_FILE);
+  return fs.accessAsync(aboutPath)
+    .catch(() => {
+      if (fs.existsSync(aboutPath)) {
+        return Promise.resolve();
+      }
+      return fs.outputFileAsync(aboutPath, ABOUT_MARKDOWN_DEFAULT);
+    });
+};
+
+/**
+ * Adds top navigation menu to default layout of site.
+ */
+Site.prototype.addTopNavToDefaultLayout = function () {
+  const siteLayoutPath = path.join(this.rootPath, LAYOUT_FOLDER_PATH);
+  const siteLayoutHeaderDefaultPath = path.join(siteLayoutPath, LAYOUT_DEFAULT_NAME, 'header.md');
+
+  return fs.outputFileAsync(siteLayoutHeaderDefaultPath, TOP_NAV_DEFAULT);
+};
+
+/**
+ * Adds a footer to default layout of site.
+ */
+Site.prototype.addFooterToDefaultLayout = function () {
+  const footerPath = path.join(this.rootPath, FOOTER_PATH);
+  const siteLayoutPath = path.join(this.rootPath, LAYOUT_FOLDER_PATH);
+  const siteLayoutFooterDefaultPath = path.join(siteLayoutPath, LAYOUT_DEFAULT_NAME, 'footer.md');
+  const wikiFooterPath = path.join(this.rootPath, WIKI_FOOTER_PATH);
+
+  return fs.accessAsync(wikiFooterPath)
+    .then(() => {
+      const footerContent = fs.readFileSync(wikiFooterPath, 'utf8');
+      const wrappedFooterContent = `<footer>\n\t${footerContent}\n</footer>`;
+      return fs.outputFileAsync(siteLayoutFooterDefaultPath, wrappedFooterContent);
+    })
+    .catch(() => {
+      if (fs.existsSync(footerPath)) {
+        return fs.copyAsync(footerPath, siteLayoutFooterDefaultPath);
+      }
+      return Promise.resolve();
+    });
+};
+
+/**
+ * Adds a site navigation bar to the default layout of the site.
+ */
+Site.prototype.addSiteNavToDefaultLayout = function () {
+  const siteLayoutPath = path.join(this.rootPath, LAYOUT_FOLDER_PATH);
+  const siteLayoutSiteNavDefaultPath = path.join(siteLayoutPath, LAYOUT_DEFAULT_NAME, 'navigation.md');
+  const wikiSiteNavPath = path.join(this.rootPath, WIKI_SITE_NAV_PATH);
+
+  return fs.accessAsync(wikiSiteNavPath)
+    .then(() => {
+      const siteNavContent = fs.readFileSync(wikiSiteNavPath, 'utf8');
+      const wrappedSiteNavContent = `<navigation>\n${siteNavContent}\n</navigation>`;
+      logger.info(`Copied over the existing _Sidebar.md file to ${path.relative(
+        this.rootPath, siteLayoutSiteNavDefaultPath)}`
+        + 'Check https://markbind.org/userGuide/tweakingThePageStructure.html#site-navigation-menus\n'
+        + 'for information on site navigation menus.');
+      return fs.outputFileSync(siteLayoutSiteNavDefaultPath, wrappedSiteNavContent);
+    })
+    .catch(() => this.buildSiteNav(siteLayoutSiteNavDefaultPath));
+};
+
+/**
+ * Builds a site navigation file from the directory structure of the site.
+ * @param siteLayoutSiteNavDefaultPath
+ */
+Site.prototype.buildSiteNav = function (siteLayoutSiteNavDefaultPath) {
+  let siteNavContent = '';
+  this.addressablePages
+    .filter(addressablePage => !addressablePage.src.startsWith('_'))
+    .forEach((page) => {
+      const addressablePagePath = path.join(this.rootPath, page.src);
+      const relativePagePathWithoutExt = FsUtil.removeExtension(
+        path.relative(this.rootPath, addressablePagePath));
+      const pageName = _.startCase(FsUtil.removeExtension(path.basename(addressablePagePath)));
+      const pageUrl = `{{ baseUrl }}/${relativePagePathWithoutExt}.html`;
+      siteNavContent += `* [${pageName}](${pageUrl})\n`;
+    });
+  const wrappedSiteNavContent = `<navigation>\n${siteNavContent}\n</navigation>`;
+  return fs.outputFileAsync(siteLayoutSiteNavDefaultPath, wrappedSiteNavContent);
+};
+
+/**
+ * Applies the default layout to all addressable pages by modifying the site config file.
+ */
+Site.prototype.addDefaultLayoutToSiteConfig = function () {
+  const configPath = path.join(this.rootPath, SITE_CONFIG_NAME);
+  return fs.readJsonAsync(configPath)
+    .then((config) => {
+      const layoutObj = { glob: '**/*.+(md|mbd)', layout: LAYOUT_DEFAULT_NAME };
+      config.pages.push(layoutObj);
+      return fs.outputJsonAsync(configPath, config);
+    });
+};
+
+Site.prototype.printBaseUrlMessage = function () {
+  logger.info('The default base URL of your site is set to /\n'
+    + 'You can change the base URL of your site by editing site.json\n'
+    + 'Check https://markbind.org/userGuide/siteConfiguration.html for more information.');
+  return Promise.resolve();
+};
+
 /**
  * Updates the paths to be traversed as addressable pages and returns a list of filepaths to be deleted
  */

src/util/fsUtil.js

@@ -24,8 +24,10 @@ module.exports = {
     return r.test(unknownPath);
   },
 
-  setExtension: (normalizedFilename, ext) => path.join(
-    path.dirname(normalizedFilename),
-    path.basename(normalizedFilename, path.extname(normalizedFilename)) + ext,
+  setExtension: (normalizedFilename, ext) => module.exports.removeExtension(normalizedFilename) + ext,
+
+  removeExtension: filePathWithExt => path.join(
+    path.dirname(filePathWithExt),
+    path.basename(filePathWithExt, path.extname(filePathWithExt)),
   ),
 };

test/functional/test.bat

@@ -2,7 +2,7 @@
 
 set sites=test_site test_site_algolia_plugin
 
-for %%a in (%sites%) do ( 
+for %%a in (%sites%) do (
 
     echo(
     echo Running %%a tests
@@ -17,5 +17,35 @@ for %%a in (%sites%) do (
     )
 )
 
+set sites_convert=test_site_convert
+
+for %%a in (%sites_convert%) do (
+
+    echo(
+    echo Running %%a tests
+
+    node ../../index.js init %%a\non_markbind_site -c
+
+    node ../../index.js build %%a\non_markbind_site
+
+    xcopy /i /q /s %%a\non_markbind_site\_site %%a\_site
+
+    node testUtil/test.js %%a
+
+    if errorlevel 1 (
+        echo Test %%a Failed
+        rmdir /s /q %%a\_site
+        rmdir /s /q %%a\non_markbind_site\_markbind
+        rmdir /s /q %%a\non_markbind_site\_site
+        del %%a\non_markbind_site\about.md %%a\non_markbind_site\index.md %%a\non_markbind_site\site.json
+        exit /b %errorlevel%
+    )
+
+    rmdir /s /q %%a\_site
+    rmdir /s /q %%a\non_markbind_site\_markbind
+    rmdir /s /q %%a\non_markbind_site\_site
+    del %%a\non_markbind_site\about.md %%a\non_markbind_site\index.md %%a\non_markbind_site\site.json
+)
+
 echo Test passed
 exit /b %errorlevel%

test/functional/test.sh

@@ -5,7 +5,7 @@ declare -a sites=("test_site" "test_site_algolia_plugin")
 for site in "${sites[@]}"
 do
     # print site name
-    echo 
+    echo
     echo "Running $site tests"
 
     # build site
@@ -21,6 +21,38 @@ do
     fi
 done
 
+function cleanup_convert {
+    # delete generated files
+    rm -rf "$site_convert"/_site
+    rm -rf "$site_convert"/non_markbind_site/_markbind "$site_convert"/non_markbind_site/_site
+    rm "$site_convert"/non_markbind_site/about.md "$site_convert"/non_markbind_site/index.md "$site_convert"/non_markbind_site/site.json
+}
+trap cleanup_convert EXIT
+
+declare -r site_convert="test_site_convert"
+
+# print site name
+echo
+echo "Running $site_convert test"
+
+# convert site
+node ../../index.js init "$site_convert"/non_markbind_site -c
+
+# build site
+node ../../index.js build "$site_convert"/non_markbind_site
+
+# copy generated site
+cp -r "$site_convert"/non_markbind_site/_site "$site_convert"
+
+# run our test script to compare html files
+node testUtil/test.js "$site_convert"
+
+if [ $? -ne 0 ]
+then
+    echo "Test result: $site FAILED"
+    exit 1
+fi
+
 # if there were no diffs
 echo "Test result: PASSED"
 exit 0