Added link checker + added check command. (#5)

Signed-off-by: Bartlomiej Plotka <[email protected]>
bwplotka · Dec 23, 2020 · db34e4a · db34e4a
1 parent d895183
commit db34e4a
Show file tree

Hide file tree

Showing 26 changed files with 1,667 additions and 156 deletions.
diff --git a/Makefile b/Makefile
@@ -56,14 +56,13 @@ deps: ## Ensures fresh go.mod and go.sum.
 .PHONY: docs
 docs: build ## Generates config snippets and doc formatting.
 	@echo ">> generating docs $(PATH)"
-	@PATH=$(GOBIN) mdox fmt *.md
+	@PATH=$(GOBIN) mdox fmt -l *.md
 
 .PHONY: format
-format: ## Formats Go code including imports and cleans up white noise.
+format: ## Formats Go code.
 format: $(GOIMPORTS)
 	@echo ">> formatting code"
 	@$(GOIMPORTS) -w $(FILES_TO_FMT)
-	@SED_BIN="$(SED)" scripts/cleanup-white-noise.sh $(FILES_TO_FMT)
 
 .PHONY: test
 test: ## Runs all Go unit tests.
@@ -90,16 +89,14 @@ lint: ## Runs various static analysis against our code.
 lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) build format docs check-git deps
 	$(call require_clean_work_tree,"detected not clean master before running lint")
 	@echo ">> verifying modules being imported"
-	@$(FAILLINT) -paths "errors=github.com/pkg/errors" ./...
+	#TODO(bwplotka): Uncomment once we upstream merrors package: @$(FAILLINT) -paths "errors=github.com/pkg/errors" ./...
 	@$(FAILLINT) -paths "fmt.{Print,PrintfPrintln,Sprint}" -ignore-tests ./...
 	@echo ">> examining all of the Go files"
 	@go vet -stdmethods=false ./...
 	@echo ">> linting all of the Go files GOGC=${GOGC}"
 	@$(GOLANGCI_LINT) run
 	@echo ">> detecting misspells"
 	@find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error
-	@echo ">> detecting white noise"
-	@find . -type f \( -name "*.md" -o -name "*.go" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh
 	@echo ">> ensuring Copyright headers"
 	@go run ./scripts/copyright/...
-	$(call require_clean_work_tree,"detected white noise or/and files without copyright; run make lint file and commit changes.")
+	$(call require_clean_work_tree,"detected files without copyright; run make lint file and commit changes.")
diff --git a/README.md b/README.md
@@ -2,70 +2,105 @@
 
 [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/bwplotka/mdox) [![Latest Release](https://img.shields.io/github/release/bwplotka/mdox.svg?style=flat-square)](https://github.com/bwplotka/mdox/releases/latest) [![CI](https://github.com/bwplotka/mdox/workflows/go/badge.svg)](https://github.com/bwplotka/mdox/actions?query=workflow%3Ago) [![Go Report Card](https://goreportcard.com/badge/github.com/bwplotka/mdox)](https://goreportcard.com/report/github.com/bwplotka/mdox)
 
-CLI toolset for maintaining automated, high quality project documentation and website leveraging markdown and git.
+`mdox` (spelled as `md docs`) is a CLI for maintaining automated, high quality project documentation and website leveraging [Github Flavored Markdown](https://github.github.com/gfm/) and git.
 
-Goal: Allow projects to have self-updating up-to-date documentation available in both markdown (e.g readable from GitHub) and static HTML. Hosted in the same repository as code, fool-proof and integrated with Pull Requests CI and hosting CD.
+## Goals
 
-### Features
+Allow projects to have self-updating up-to-date documentation available in both markdown (e.g readable from GitHub) and static HTML. Hosted in the same repository as code and integrated with Pull Requests CI, hosting CD and code generation.
 
-```bash mdox-gen-exec="mdox --help"
-usage: mdox [<flags>] <command> [<args> ...]
+## Features
 
-Markdown Project Documentation Toolbox.
+* Enhanced amd consistent formatting for markdown files, focused on `.md` code readability.
+* Auto generation of code block content based on `mdoc-exec` directives (see [#code-generation](#code-generation)).
+* Robust and fast relative and remote link checking.
+* "Localizing" links to relative docs if specified (useful for multi-domain websites or multi-version doc).
 
-Flags:
-  -h, --help               Show context-sensitive help (also try --help-long and
-                           --help-man).
-      --version            Show application version.
-      --log.level=info     Log filtering level.
-      --log.format=logfmt  Log format to use. Possible options: logfmt or json.
+## Usage
+
+Just run `mdox fmt` and pass markdown files (or glob matching those).
 
-Commands:
-  help [<command>...]
-    Show help.
+For example this README is formatted by the CI on every PR using [`mdox fmt -l *.md` command](https://github.com/bwplotka/mdox/blob/9e183714070f464b1ef089da3df8048aff1abeda/Makefile#L59).
 
-  fmt <files>...
-    Formats given markdown files uniformly following GFM (Github Flavored
-    Markdown: https://github.github.com/gfm/).
+```bash mdox-gen-exec="mdox fmt --help"
+usage: mdox fmt [<flags>] <files>...
 
-    Additionally it supports special fenced code directives to autogenerate code
-    snippets:
+Formats in-place given markdown files uniformly following GFM (Github Flavored
+Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md
 
-      ```<lang> mdox-gen-exec="<executable + arguments>"
+Flags:
+  -h, --help                     Show context-sensitive help (also try
+                                 --help-long and --help-man).
+      --version                  Show application version.
+      --log.level=info           Log filtering level.
+      --log.format=logfmt        Log format to use. Possible options: logfmt or
+                                 json.
+      --check                    If true, fmt will not modify the given files,
+                                 instead it will fail if files needs formatting
+      --code.disable-directives  If false, fmt will parse custom fenced code
+                                 directives prefixed with 'mdox-gen' to
+                                 autogenerate code snippets. For example:
+
+                                   ```<lang> mdox-gen-exec="<executable + arguments>"
+                                 
+                                 This directive runs executable with arguments
+                                 and put its stderr and stdout output inside
+                                 code block content, replacing existing one.
+      --links.anchor-dir=LINKS.ANCHOR-DIR  
+                                 Anchor directory for link transformers. PWD
+                                 flag is not specified.
+      --links.localise.address-regex=LINKS.LOCALISE.ADDRESS-REGEX  
+                                 If specified, all HTTP(s) links that target a
+                                 domain and path matching given regexp will be
+                                 transformed to relative to anchor dir path (if
+                                 exists).Absolute path links will be converted
+                                 to relative links to anchor dri as well.
+  -l, --links.validate           If true, all links will be validated
+      --links.validate.address-regex=^$  
+                                 If specified, all links will be validated,
+                                 except those matching the given target address.
+
+Args:
+  <files>  Markdown file(s) to process.
 
-    This directive runs executable with arguments and put its stderr and stdout
-    output inside code block content, replacing existing one.
+```
 
-    Example: mdox fmt *.md
+### Code Generation
 
-  web gen <files>...
-    Generate versioned docs
+It's not uncommon that documentation is explaining code or configuration snippets. One of the challenges of such documentation is keeping it up to date. This is where `mdox` code block directives comes handy! To ensure mdox will auto update code snippet add `mdox-gen-exec="<whatever command you want take output from>"` after language directive on code block.
 
+For example this Readme contains `mdox --help` which is has to be auto generated on every PR:
 
+```markdown
+``` bash mdox-gen-exec="mdox fmt --help"
+...
 ```
 
-### Production Usage
+You can disable this feature by specifying `--code.disable-directives`
 
-* [Thanos](https://github.com/bwplotka/thanos) (TBD)
+### Installing
 
-## Requirements
+Requirements to build this tool:
 
-* Go 1.14+
+* Go 1.15+
 * Linux or MacOS
 
-## Installing
-
 ```shell
 go get github.com/bwplotka/mdox && go mod tidy
 ```
 
-or via [bingo](github.com/bwplotka/bingo) if want to pin it:
+or via [bingo](https://github.com/bwplotka/bingo) if want to pin it:
 
 ```shell
 bingo get -u github.com/bwplotka/mdox
 ```
 
-Any contributions are welcome! Just use GitHub Issues and Pull Requests as usual. We follow [Thanos Go coding style](https://thanos.io/contributing/coding-style-guide.md/) guide.
+### Production Usage
+
+* [Thanos](https://github.com/bwplotka/thanos) (TBD)
+
+## Contributing
+
+Any contributions are welcome! Just use GitHub Issues and Pull Requests as usual. We follow [Thanos Go coding style](https://thanos.io/tip/contributing/coding-style-guide.md/) guide.
 
 ## Initial Author
 

diff --git a/go.mod b/go.mod
@@ -3,19 +3,20 @@ module github.com/bwplotka/mdox
 go 1.14
 
 require (
-	github.com/Kunde21/markdownfmt/v2 v2.0.2
+	github.com/Kunde21/markdownfmt/v2 v2.0.4-0.20201214081534-353201c4cdce
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/go-kit/kit v0.10.0
+	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gohugoio/hugo v0.74.3
-	github.com/google/go-cmp v0.4.0 // indirect
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mitchellh/mapstructure v1.2.2 // indirect
 	github.com/oklog/run v1.1.0
 	github.com/pkg/errors v0.9.1
+	github.com/sergi/go-diff v1.0.0
+	github.com/stretchr/testify v1.5.1
 	github.com/yuin/goldmark v1.2.1
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
 	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 )
 
-replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201129164736-749754008490
+replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201223130030-c496cb0bcc88