From db34e4aff585fd4c2d35219160c868d4f657e3e5 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 23 Dec 2020 23:51:03 +0000 Subject: [PATCH] Added link checker + added check command. (#5) Signed-off-by: Bartlomiej Plotka --- Makefile | 11 +- README.md | 103 ++++-- go.mod | 9 +- go.sum | 58 +++- main.go | 83 ++++- pkg/gitdiff/print.go | 219 +++++++++++++ pkg/mdformatter/linktransformer/link.go | 297 ++++++++++++++++++ pkg/mdformatter/linktransformer/link_test.go | 235 ++++++++++++++ pkg/mdformatter/mdformatter.go | 105 ++++++- pkg/mdformatter/mdformatter_test.go | 31 +- pkg/{ => mdformatter}/mdgen/mdgen.go | 4 +- pkg/{ => mdformatter}/mdgen/mdgen_test.go | 0 pkg/{ => mdformatter}/mdgen/testdata/cfg.go | 0 .../mdgen/testdata/mdgen_formatted.md | 0 .../mdgen/testdata/mdgen_not_formatted.md | 0 pkg/{ => mdformatter}/mdgen/testdata/out.sh | 0 pkg/{ => mdformatter}/mdgen/testdata/out2.sh | 0 pkg/mdformatter/testdata/formatted.md | 2 + .../testdata/formatted_and_transformed.md | 2 + pkg/mdformatter/testdata/not_formatted.md | 3 + .../testdata/not_formatted.md.diff | 150 +++++++++ pkg/mdformatter/transformer.go | 55 +++- pkg/merrors/errors.go | 172 ++++++++++ pkg/merrors/errors_test.go | 223 +++++++++++++ pkg/runutil/runutil.go | 57 +--- scripts/cleanup-white-noise.sh | 4 - 26 files changed, 1667 insertions(+), 156 deletions(-) create mode 100644 pkg/gitdiff/print.go create mode 100644 pkg/mdformatter/linktransformer/link.go create mode 100644 pkg/mdformatter/linktransformer/link_test.go rename pkg/{ => mdformatter}/mdgen/mdgen.go (95%) rename pkg/{ => mdformatter}/mdgen/mdgen_test.go (100%) rename pkg/{ => mdformatter}/mdgen/testdata/cfg.go (100%) rename pkg/{ => mdformatter}/mdgen/testdata/mdgen_formatted.md (100%) rename pkg/{ => mdformatter}/mdgen/testdata/mdgen_not_formatted.md (100%) rename pkg/{ => mdformatter}/mdgen/testdata/out.sh (100%) rename pkg/{ => mdformatter}/mdgen/testdata/out2.sh (100%) create mode 100644 pkg/mdformatter/testdata/not_formatted.md.diff create mode 100644 pkg/merrors/errors.go create mode 100644 pkg/merrors/errors_test.go delete mode 100755 scripts/cleanup-white-noise.sh diff --git a/Makefile b/Makefile index dad8e00..2aaf3b5 100644 --- a/Makefile +++ b/Makefile @@ -56,14 +56,13 @@ deps: ## Ensures fresh go.mod and go.sum. .PHONY: docs docs: build ## Generates config snippets and doc formatting. @echo ">> generating docs $(PATH)" - @PATH=$(GOBIN) mdox fmt *.md + @PATH=$(GOBIN) mdox fmt -l *.md .PHONY: format -format: ## Formats Go code including imports and cleans up white noise. +format: ## Formats Go code. format: $(GOIMPORTS) @echo ">> formatting code" @$(GOIMPORTS) -w $(FILES_TO_FMT) - @SED_BIN="$(SED)" scripts/cleanup-white-noise.sh $(FILES_TO_FMT) .PHONY: test test: ## Runs all Go unit tests. @@ -90,7 +89,7 @@ lint: ## Runs various static analysis against our code. lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) build format docs check-git deps $(call require_clean_work_tree,"detected not clean master before running lint") @echo ">> verifying modules being imported" - @$(FAILLINT) -paths "errors=github.com/pkg/errors" ./... + #TODO(bwplotka): Uncomment once we upstream merrors package: @$(FAILLINT) -paths "errors=github.com/pkg/errors" ./... @$(FAILLINT) -paths "fmt.{Print,PrintfPrintln,Sprint}" -ignore-tests ./... @echo ">> examining all of the Go files" @go vet -stdmethods=false ./... @@ -98,8 +97,6 @@ lint: $(FAILLINT) $(GOLANGCI_LINT) $(MISSPELL) build format docs check-git deps @$(GOLANGCI_LINT) run @echo ">> detecting misspells" @find . -type f | grep -v vendor/ | grep -vE '\./\..*' | xargs $(MISSPELL) -error - @echo ">> detecting white noise" - @find . -type f \( -name "*.md" -o -name "*.go" \) | SED_BIN="$(SED)" xargs scripts/cleanup-white-noise.sh @echo ">> ensuring Copyright headers" @go run ./scripts/copyright/... - $(call require_clean_work_tree,"detected white noise or/and files without copyright; run make lint file and commit changes.") + $(call require_clean_work_tree,"detected files without copyright; run make lint file and commit changes.") diff --git a/README.md b/README.md index cdbd600..d1ba48f 100644 --- a/README.md +++ b/README.md @@ -2,70 +2,105 @@ [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/bwplotka/mdox) [![Latest Release](https://img.shields.io/github/release/bwplotka/mdox.svg?style=flat-square)](https://github.com/bwplotka/mdox/releases/latest) [![CI](https://github.com/bwplotka/mdox/workflows/go/badge.svg)](https://github.com/bwplotka/mdox/actions?query=workflow%3Ago) [![Go Report Card](https://goreportcard.com/badge/github.com/bwplotka/mdox)](https://goreportcard.com/report/github.com/bwplotka/mdox) -CLI toolset for maintaining automated, high quality project documentation and website leveraging markdown and git. +`mdox` (spelled as `md docs`) is a CLI for maintaining automated, high quality project documentation and website leveraging [Github Flavored Markdown](https://github.github.com/gfm/) and git. -Goal: Allow projects to have self-updating up-to-date documentation available in both markdown (e.g readable from GitHub) and static HTML. Hosted in the same repository as code, fool-proof and integrated with Pull Requests CI and hosting CD. +## Goals -### Features +Allow projects to have self-updating up-to-date documentation available in both markdown (e.g readable from GitHub) and static HTML. Hosted in the same repository as code and integrated with Pull Requests CI, hosting CD and code generation. -```bash mdox-gen-exec="mdox --help" -usage: mdox [] [ ...] +## Features -Markdown Project Documentation Toolbox. +* Enhanced amd consistent formatting for markdown files, focused on `.md` code readability. +* Auto generation of code block content based on `mdoc-exec` directives (see [#code-generation](#code-generation)). +* Robust and fast relative and remote link checking. +* "Localizing" links to relative docs if specified (useful for multi-domain websites or multi-version doc). -Flags: - -h, --help Show context-sensitive help (also try --help-long and - --help-man). - --version Show application version. - --log.level=info Log filtering level. - --log.format=logfmt Log format to use. Possible options: logfmt or json. +## Usage + +Just run `mdox fmt` and pass markdown files (or glob matching those). -Commands: - help [...] - Show help. +For example this README is formatted by the CI on every PR using [`mdox fmt -l *.md` command](https://github.com/bwplotka/mdox/blob/9e183714070f464b1ef089da3df8048aff1abeda/Makefile#L59). - fmt ... - Formats given markdown files uniformly following GFM (Github Flavored - Markdown: https://github.github.com/gfm/). +```bash mdox-gen-exec="mdox fmt --help" +usage: mdox fmt [] ... - Additionally it supports special fenced code directives to autogenerate code - snippets: +Formats in-place given markdown files uniformly following GFM (Github Flavored +Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md - ``` mdox-gen-exec="" +Flags: + -h, --help Show context-sensitive help (also try + --help-long and --help-man). + --version Show application version. + --log.level=info Log filtering level. + --log.format=logfmt Log format to use. Possible options: logfmt or + json. + --check If true, fmt will not modify the given files, + instead it will fail if files needs formatting + --code.disable-directives If false, fmt will parse custom fenced code + directives prefixed with 'mdox-gen' to + autogenerate code snippets. For example: + + ``` mdox-gen-exec="" + + This directive runs executable with arguments + and put its stderr and stdout output inside + code block content, replacing existing one. + --links.anchor-dir=LINKS.ANCHOR-DIR + Anchor directory for link transformers. PWD + flag is not specified. + --links.localise.address-regex=LINKS.LOCALISE.ADDRESS-REGEX + If specified, all HTTP(s) links that target a + domain and path matching given regexp will be + transformed to relative to anchor dir path (if + exists).Absolute path links will be converted + to relative links to anchor dri as well. + -l, --links.validate If true, all links will be validated + --links.validate.address-regex=^$ + If specified, all links will be validated, + except those matching the given target address. + +Args: + Markdown file(s) to process. - This directive runs executable with arguments and put its stderr and stdout - output inside code block content, replacing existing one. +``` - Example: mdox fmt *.md +### Code Generation - web gen ... - Generate versioned docs +It's not uncommon that documentation is explaining code or configuration snippets. One of the challenges of such documentation is keeping it up to date. This is where `mdox` code block directives comes handy! To ensure mdox will auto update code snippet add `mdox-gen-exec=""` after language directive on code block. +For example this Readme contains `mdox --help` which is has to be auto generated on every PR: +```markdown +``` bash mdox-gen-exec="mdox fmt --help" +... ``` -### Production Usage +You can disable this feature by specifying `--code.disable-directives` -* [Thanos](https://github.com/bwplotka/thanos) (TBD) +### Installing -## Requirements +Requirements to build this tool: -* Go 1.14+ +* Go 1.15+ * Linux or MacOS -## Installing - ```shell go get github.com/bwplotka/mdox && go mod tidy ``` -or via [bingo](github.com/bwplotka/bingo) if want to pin it: +or via [bingo](https://github.com/bwplotka/bingo) if want to pin it: ```shell bingo get -u github.com/bwplotka/mdox ``` -Any contributions are welcome! Just use GitHub Issues and Pull Requests as usual. We follow [Thanos Go coding style](https://thanos.io/contributing/coding-style-guide.md/) guide. +### Production Usage + +* [Thanos](https://github.com/bwplotka/thanos) (TBD) + +## Contributing + +Any contributions are welcome! Just use GitHub Issues and Pull Requests as usual. We follow [Thanos Go coding style](https://thanos.io/tip/contributing/coding-style-guide.md/) guide. ## Initial Author diff --git a/go.mod b/go.mod index f0d3852..d617417 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,20 @@ module github.com/bwplotka/mdox go 1.14 require ( - github.com/Kunde21/markdownfmt/v2 v2.0.2 + github.com/Kunde21/markdownfmt/v2 v2.0.4-0.20201214081534-353201c4cdce github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/go-kit/kit v0.10.0 + github.com/gocolly/colly/v2 v2.1.0 github.com/gohugoio/hugo v0.74.3 - github.com/google/go-cmp v0.4.0 // indirect github.com/mattn/go-shellwords v1.0.10 github.com/mitchellh/mapstructure v1.2.2 // indirect github.com/oklog/run v1.1.0 github.com/pkg/errors v0.9.1 + github.com/sergi/go-diff v1.0.0 + github.com/stretchr/testify v1.5.1 github.com/yuin/goldmark v1.2.1 - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) -replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201129164736-749754008490 +replace github.com/Kunde21/markdownfmt/v2 => github.com/bwplotka/markdownfmt/v2 v2.0.0-20201223130030-c496cb0bcc88 diff --git a/go.sum b/go.sum index ebc7145..9d04684 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtix github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190418212003-6ac0b49e7197/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -51,6 +53,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= +github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= +github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= +github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= +github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= +github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= +github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= +github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -75,8 +87,8 @@ github.com/bep/gitmap v1.1.2/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kY github.com/bep/golibsass v0.6.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bwplotka/markdownfmt/v2 v2.0.0-20201129164736-749754008490 h1:wP/QJown7dFXZp5N4HCNk39iDWkczn4F8noYqA8Wcbg= -github.com/bwplotka/markdownfmt/v2 v2.0.0-20201129164736-749754008490/go.mod h1:niFn22lPHG2owQ+pHRB0bz3tkrCuVjvlUy4iFdRY+Bo= +github.com/bwplotka/markdownfmt/v2 v2.0.0-20201223130030-c496cb0bcc88 h1:v5CH+iRtJ8rKDp5/tIqrQKMjS5hutLVnJGWc6Im83Xw= +github.com/bwplotka/markdownfmt/v2 v2.0.0-20201223130030-c496cb0bcc88/go.mod h1:niFn22lPHG2owQ+pHRB0bz3tkrCuVjvlUy4iFdRY+Bo= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -149,6 +161,10 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= +github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= +github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -160,11 +176,21 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -226,6 +252,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jdkato/prose v1.1.1/go.mod h1:jkF0lkxaX5PFSlk9l4Gh9Y+T57TqUZziWT7uZbW5ADg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -238,6 +265,8 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -372,9 +401,12 @@ github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6/go.mod h1:J github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -406,6 +438,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -416,6 +449,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tdewolff/minify/v2 v2.6.2/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w= github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= +github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= +github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -464,6 +499,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -475,6 +511,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -499,6 +536,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -556,6 +595,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -572,6 +612,8 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -580,6 +622,7 @@ google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -590,6 +633,17 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index dbedc0a..aef751f 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,8 @@ import ( "github.com/bwplotka/mdox/pkg/extkingpin" "github.com/bwplotka/mdox/pkg/mdformatter" - "github.com/bwplotka/mdox/pkg/mdgen" + "github.com/bwplotka/mdox/pkg/mdformatter/linktransformer" + "github.com/bwplotka/mdox/pkg/mdformatter/mdgen" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/oklog/run" @@ -44,8 +45,7 @@ func setupLogger(logLevel, logFormat string) log.Logger { if logFormat == logFormatJson { logger = log.NewJSONLogger(log.NewSyncWriter(os.Stderr)) } - logger = level.NewFilter(logger, lvl) - return log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller) + return level.NewFilter(logger, lvl) } func main() { @@ -81,8 +81,12 @@ func main() { } if err := g.Run(); err != nil { - // Use %+v for github.com/pkg/errors error to print with stack. - level.Error(logger).Log("err", fmt.Sprintf("%+v", errors.Wrapf(err, "%s command failed", cmd))) + if *logLevel == "debug" { + // Use %+v for github.com/pkg/errors error to print with stack. + level.Error(logger).Log("err", fmt.Sprintf("%+v", errors.Wrapf(err, "%s command failed", cmd))) + os.Exit(1) + } + level.Error(logger).Log("err", errors.Wrapf(err, "%s command failed", cmd)) os.Exit(1) } level.Info(logger).Log("msg", "exiting") @@ -101,20 +105,65 @@ func interrupt(logger log.Logger, cancel <-chan struct{}) error { } func registerFmt(_ context.Context, app *extkingpin.App) { - cmd := app.Command("fmt", ` -Formats given markdown files uniformly following GFM (Github Flavored Markdown: https://github.github.com/gfm/). - -Additionally it supports special fenced code directives to autogenerate code snippets: + cmd := app.Command("fmt", "Formats in-place given markdown files uniformly following GFM (Github Flavored Markdown: https://github.github.com/gfm/). Example: mdox fmt *.md") + files := cmd.Arg("files", "Markdown file(s) to process.").Required().ExistingFiles() + checkOnly := cmd.Flag("check", "If true, fmt will not modify the given files, instead it will fail if files needs formatting").Bool() + disableGenCodeBlocksDirectives := cmd.Flag("code.disable-directives", `If false, fmt will parse custom fenced code directives prefixed with 'mdox-gen' to autogenerate code snippets. For example: `+"```"+` mdox-gen-exec="" - -This directive runs executable with arguments and put its stderr and stdout output inside code block content, replacing existing one. - -Example: mdox fmt *.md -`) - files := cmd.Arg("files", "Markdown file(s) to process.").Required().ExistingFiles() - cmd.Run(func(ctx context.Context, logger log.Logger) error { - return mdformatter.Format(ctx, logger, *files, mdformatter.WithCodeBlockTransformer(mdgen.NewCodeBlockTransformer())) +This directive runs executable with arguments and put its stderr and stdout output inside code block content, replacing existing one.`).Bool() + linksAnchorDir := cmd.Flag("links.anchor-dir", "Anchor directory for link transformers. PWD flag is not specified.").ExistingDir() + linksLocaliseForAddress := cmd.Flag("links.localise.address-regex", "If specified, all HTTP(s) links that target a domain and path matching given regexp will be transformed to relative to anchor dir path (if exists)."+ + "Absolute path links will be converted to relative links to anchor dri as well.").Regexp() + // TODO(bwplotka): Add cache in file? + linksValidateEnabled := cmd.Flag("links.validate", "If true, all links will be validated").Short('l').Bool() + linksValidateExceptDomains := cmd.Flag("links.validate.address-regex", "If specified, all links will be validated, except those matching the given target address.").Default(`^$`).Regexp() + + cmd.Run(func(ctx context.Context, logger log.Logger) (err error) { + var opts []mdformatter.Option + if !*disableGenCodeBlocksDirectives { + opts = append(opts, mdformatter.WithCodeBlockTransformer(mdgen.NewCodeBlockTransformer())) + } + + if len(*files) == 0 { + return errors.New("no files to format") + } + + for i := range *files { + (*files)[i], err = filepath.Abs((*files)[i]) + if err != nil { + return err + } + } + + anchorDir, err := linktransformer.GetAnchorDir(*linksAnchorDir, *files) + if err != nil { + return err + } + + var linkTr []mdformatter.LinkTransformer + if *linksLocaliseForAddress != nil { + linkTr = append(linkTr, linktransformer.NewLocalizer(logger, *linksLocaliseForAddress, anchorDir)) + } + if *linksValidateEnabled { + linkTr = append(linkTr, linktransformer.NewValidator(logger, *linksValidateExceptDomains, anchorDir)) + } + + if len(linkTr) > 0 { + opts = append(opts, mdformatter.WithLinkTransformer(linktransformer.NewChain(linkTr...))) + } + + if *checkOnly { + diff, err := mdformatter.IsFormatted(ctx, logger, *files, opts...) + if err != nil { + return err + } + if len(diff) == 0 { + return nil + } + return errors.Errorf("files not formatted: %v", diff.String()) + } + return mdformatter.Format(ctx, logger, *files, opts...) }) } diff --git a/pkg/gitdiff/print.go b/pkg/gitdiff/print.go new file mode 100644 index 0000000..5c8c2d5 --- /dev/null +++ b/pkg/gitdiff/print.go @@ -0,0 +1,219 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +// gitdiff is an adaptation of https://github.com/sourcegraph/go-diff/blob/master/diff/print.go that prints popular diffmatchpatch.Diff diffs. + +package gitdiff + +import ( + "bytes" + "fmt" + "strings" + "unsafe" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +type Diff struct { + diffs []diffmatchpatch.Diff + aFn, bFn string +} + +func yoloString(b []byte) string { + return *((*string)(unsafe.Pointer(&b))) +} + +func CompareBytes(a []byte, aFn string, b []byte, bFn string) Diff { + return Compare(yoloString(a), aFn, yoloString(b), bFn) +} + +func Compare(a, aFn, b, bFn string) Diff { + dmp := diffmatchpatch.New() + return Diff{ + diffs: DiffLines(dmp.DiffMain(a, b, true)), + aFn: aFn, + bFn: bFn, + } +} + +// CombineIntoLines traverse through diff and creates separate per line diff for each prefix and suffix diff chunks. +// NOTE: This is useful to normalize output to git diff. +func DiffLines(diff []diffmatchpatch.Diff) (ret []diffmatchpatch.Diff) { + // TODO(bwplotka): Everything is per line now, but we could merge same op lines. But... whatever (: + var rollingDelLine, rollingAddLine string + for _, d := range diff { + for i, line := range strings.Split(d.Text, "\n") { + if i > 0 { + switch d.Type { + case diffmatchpatch.DiffEqual: + if rollingAddLine == rollingDelLine { + ret = append(ret, diffmatchpatch.Diff{Type: d.Type, Text: rollingAddLine}) + rollingAddLine, rollingDelLine = "", "" + break + } + ret = append(ret, diffmatchpatch.Diff{Type: diffmatchpatch.DiffDelete, Text: rollingDelLine}) + ret = append(ret, diffmatchpatch.Diff{Type: diffmatchpatch.DiffInsert, Text: rollingAddLine}) + rollingAddLine, rollingDelLine = "", "" + case diffmatchpatch.DiffInsert: + ret = append(ret, diffmatchpatch.Diff{Type: d.Type, Text: rollingAddLine}) + rollingAddLine = "" + case diffmatchpatch.DiffDelete: + ret = append(ret, diffmatchpatch.Diff{Type: d.Type, Text: rollingDelLine}) + rollingDelLine = "" + } + } + + switch d.Type { + case diffmatchpatch.DiffEqual: + rollingDelLine += line + rollingAddLine += line + case diffmatchpatch.DiffInsert: + rollingAddLine += line + case diffmatchpatch.DiffDelete: + rollingDelLine += line + } + } + } + + if rollingAddLine == rollingDelLine { + return append(ret, diffmatchpatch.Diff{Type: diffmatchpatch.DiffEqual, Text: rollingAddLine}) + } + ret = append(ret, diffmatchpatch.Diff{Type: diffmatchpatch.DiffDelete, Text: rollingDelLine}) + ret = append(ret, diffmatchpatch.Diff{Type: diffmatchpatch.DiffInsert, Text: rollingAddLine}) + return ret +} + +// ToCombinedFormat prints diff in git combined diff format, specified in https://git-scm.com/docs/diff-format#_combined_diff_format. +func (d Diff) ToCombinedFormat() []byte { + const contextLines = 3 + + var buf bytes.Buffer + _, _ = fmt.Fprintln(&buf, "---", d.aFn) + _, _ = fmt.Fprintln(&buf, "+++", d.bFn) + _, _ = buf.Write(PrintDMPDiff(d.diffs, contextLines)) + return buf.Bytes() +} + +type entry struct { + preLines, postLines []string + buf bytes.Buffer + aRef, bRef int + adds, dels int +} + +func (e *entry) reset() { + e.preLines = e.preLines[:0] + e.postLines = e.postLines[:0] + e.buf.Reset() + e.aRef, e.bRef, e.adds, e.dels = 0, 0, 0, 0 +} + +func (e *entry) started() bool { + return e.adds+e.dels > 0 +} + +func (e *entry) finish(w *bytes.Buffer, contextLines int) { + if !e.started() { + return + } + _, _ = fmt.Fprintf(w, "@@ -%d,%d +%d,%d @@\n", e.aRef, e.dels, e.bRef, e.adds) + _, _ = w.Write(e.buf.Bytes()) + if contextLines > len(e.postLines) { + contextLines = len(e.postLines) + } + writeLines(w, e.postLines[:contextLines], ' ') + e.reset() +} + +func (e *entry) addChange(lines []string, op diffmatchpatch.Operation) { + if len(lines) == 0 { + return + } + + if len(e.preLines) > 0 { + writeLines(&e.buf, e.preLines, ' ') + e.preLines = e.preLines[:0] + } + // Post lines if any, no becomes "mid lines" + if len(e.postLines) > 0 { + writeLines(&e.buf, e.postLines, ' ') + e.postLines = e.postLines[:0] + } + + switch op { + case diffmatchpatch.DiffInsert: + e.adds += len(lines) + writeLines(&e.buf, lines, '+') + case diffmatchpatch.DiffDelete: + e.dels += len(lines) + writeLines(&e.buf, lines, '-') + } +} + +func writeLines(b *bytes.Buffer, lines []string, sign byte) { + for _, l := range lines { + if l != "" || sign != ' ' { + _ = b.WriteByte(sign) + } + _, _ = b.WriteString(l) + _ = b.WriteByte('\n') + } +} + +// PrintDMPDiff prints diffmatchpatch.Diff slice in git combined diff format, specified in https://git-scm.com/docs/diff-format#_combined_diff_format. +// It's caller responsibility to add extended header lines and git diff header if needed. +func PrintDMPDiff(diff []diffmatchpatch.Diff, contextLines int) []byte { + var ( + buf bytes.Buffer + e = &entry{} + + aLinesCount, bLinesCount int + ) + + for _, d := range diff { + lines := strings.Split(d.Text, "\n") + switch d.Type { + case diffmatchpatch.DiffEqual: + aLinesCount += len(lines) + bLinesCount += len(lines) + if e.started() { + // Take 2x context lines, so we are sure continue one entry if changes are within similar place. + needed := 2*contextLines - len(e.postLines) + if needed > len(lines) { + needed = len(lines) + } + + e.postLines = append(e.postLines, lines[:needed]...) + if len(e.postLines) == 2*contextLines { + e.finish(&buf, contextLines) + } + } + + if !e.started() { + end := len(lines) - contextLines + cut := 0 + if end < 0 { + cut = len(e.preLines) + end + if cut < 0 { + cut = 0 + } + end = 0 + } + e.preLines = append(e.preLines[cut:], lines[end:]...) + e.aRef = aLinesCount - len(e.preLines) + e.bRef = bLinesCount - len(e.preLines) + } + case diffmatchpatch.DiffInsert: + bLinesCount += len(lines) + e.addChange(lines, d.Type) + case diffmatchpatch.DiffDelete: + aLinesCount += len(lines) + e.addChange(lines, d.Type) + } + } + + if e.started() { + e.finish(&buf, contextLines) + } + return buf.Bytes() +} diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go new file mode 100644 index 0000000..b637456 --- /dev/null +++ b/pkg/mdformatter/linktransformer/link.go @@ -0,0 +1,297 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package linktransformer + +import ( + "bufio" + "bytes" + "context" + "io" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/bwplotka/mdox/pkg/mdformatter" + "github.com/bwplotka/mdox/pkg/merrors" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/gocolly/colly/v2" + "github.com/pkg/errors" +) + +var remoteLinkPrefixRe = regexp.MustCompile(`^http[s]?://`) + +type chain struct { + chain []mdformatter.LinkTransformer +} + +func NewChain(c ...mdformatter.LinkTransformer) mdformatter.LinkTransformer { + return &chain{chain: c} +} + +func (l *chain) TransformDestination(ctx context.Context, docPath string, destination []byte) (_ []byte, err error) { + for _, c := range l.chain { + destination, err = c.TransformDestination(ctx, docPath, destination) + if err != nil { + return nil, err + } + } + return destination, nil +} + +func (l *chain) Close() error { + errs := merrors.New() + for _, c := range l.chain { + errs.Add(c.Close()) + } + return errs.Err() +} + +type localizer struct { + address *regexp.Regexp + anchorDir string + + localLinksByFile localLinksCache + + logger log.Logger +} + +// NewLocalizer returns mdformatter.LinkTransformer that transforms links that matches address via given regexp to local markdown file path (if exists). +func NewLocalizer(logger log.Logger, address *regexp.Regexp, anchorDir string) mdformatter.LinkTransformer { + return &localizer{logger: logger, address: address, anchorDir: anchorDir, localLinksByFile: map[string]*[]string{}} +} + +func (l *localizer) TransformDestination(_ context.Context, docPath string, destination []byte) (_ []byte, err error) { + matches := remoteLinkPrefixRe.FindAllIndex(destination, 1) + if matches != nil { + // URLs. Remove http/https prefix. + newDest := string(destination[matches[0][1]:]) + // NOTE: We don't check if passed regexp does not make sense (it's empty string etc). + matches = l.address.FindAllStringIndex(newDest, 1) + if matches == nil { + return destination, nil + } + + // Remove matched address. + newDest = filepath.Join(l.anchorDir, newDest[matches[0][1]:]) + if err := l.localLinksByFile.Lookup(newDest); err != nil { + level.Debug(l.logger).Log("msg", "attempted localization failed, no such local link; skipping", "err", err) + return destination, nil + } + // NOTE: This assumes GetAnchorDir was used, so we validated if docPath is in the path of anchorDir. + return absLinkToRelLink(newDest, docPath) + } + + // Relative or absolute path. + newDest := absLocalLink(l.anchorDir, docPath, string(destination)) + + if err := l.localLinksByFile.Lookup(newDest); err != nil { + level.Debug(l.logger).Log("msg", "attempted localization failed, no such local link; skipping", "err", err) + return destination, nil + } + // NOTE: This assumes GetAnchorDir was used, so we validated if docPath is in the path of anchorDir. + return absLinkToRelLink(newDest, docPath) +} + +func (l *localizer) Close() error { + return nil +} + +type validator struct { + localLinksByFile localLinksCache + anchorDir string + + except *regexp.Regexp + c *colly.Collector + + errs *merrors.NilOrMultiError + logger log.Logger +} + +// NewValidator returns mdformatter.LinkTransformer that crawls all links. +func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) mdformatter.LinkTransformer { + c := colly.NewCollector(colly.Async()) + errs := merrors.New() + c.OnError(func(response *colly.Response, err error) { + errs.Add(errors.Wrapf(err, "link %q; status code %v", response.Request.URL.String(), response.StatusCode)) + }) + return &validator{logger: logger, c: c, errs: errs, except: except, localLinksByFile: map[string]*[]string{}, anchorDir: anchorDir} +} + +func (l *validator) TransformDestination(_ context.Context, docPath string, destination []byte) (_ []byte, err error) { + if l.except.Match(destination) { + return destination, nil + } + + matches := remoteLinkPrefixRe.FindAllIndex(destination, 1) + if matches == nil { + // Relative or absolute path. Check if exists. + newDest := absLocalLink(l.anchorDir, docPath, string(destination)) + + // Local link. Check if exists. + if err := l.localLinksByFile.Lookup(newDest); err != nil { + l.errs.Add(errors.Wrapf(err, "link %v, normalized to %v", string(destination), newDest)) + } + return destination, nil + } + + // TODO(bwplotka): Respect context. + if err := l.c.Visit(string(destination)); err != nil && err != colly.ErrAlreadyVisited { + l.errs.Add(errors.Wrapf(err, "remote link %v", string(destination))) + } + return destination, nil +} + +func (l *validator) Close() error { + l.c.Wait() + + if err := l.errs.Err(); err != nil { + for _, e := range err.Errors() { + level.Warn(l.logger).Log("msg", e.Error()) + } + return errors.Errorf("found %v problems with links.", len(err.Errors())) + } + return nil +} + +type localLinksCache map[string]*[]string + +type LookupError error + +var ( + FileNotFoundErr = LookupError(errors.New("file not found")) + IDNotFoundErr = LookupError(errors.New("file exists, but does not have such id")) +) + +func absLocalLink(anchorDir string, docPath string, destination string) string { + newDest := destination + switch { + case filepath.IsAbs(destination): + return filepath.Join(anchorDir, destination[1:]) + case destination == ".": + newDest = filepath.Base(docPath) + case strings.HasPrefix(destination, "#"): + newDest = filepath.Base(docPath) + destination + } + return filepath.Join(filepath.Dir(docPath), newDest) +} + +func absLinkToRelLink(absLink string, docPath string) ([]byte, error) { + absLinkSplit := strings.Split(absLink, "#") + rel, err := filepath.Rel(filepath.Dir(docPath), absLinkSplit[0]) + if err != nil { + return nil, err + } + + if rel == filepath.Base(docPath) { + rel = "." + } + + if len(absLinkSplit) == 1 { + return []byte(rel), nil + } + + if rel != "." { + return append([]byte(rel), append([]byte{'#'}, absLinkSplit[1]...)...), nil + } + return append([]byte{'#'}, absLinkSplit[1]...), nil +} + +// Lookup looks for given link in local anchorDir. It returns error if link can't be found. +func (l localLinksCache) Lookup(absLink string) error { + absLinkSplit := strings.Split(absLink, "#") + ids, ok := l[absLinkSplit[0]] + if !ok { + if err := l.addRelLinks(absLinkSplit[0]); err != nil { + return err + } + ids = l[absLinkSplit[0]] + } + if ids == nil { + return errors.Wrapf(FileNotFoundErr, "%v", absLinkSplit[0]) + } + + if len(absLinkSplit) == 1 { + return nil + } + + for _, id := range *ids { + if strings.Compare(id, absLinkSplit[1]) == 0 { + return nil + } + } + return errors.Wrapf(IDNotFoundErr, "link %v, existing ids: %v", absLink, *ids) +} + +func (l localLinksCache) addRelLinks(localFile string) error { + // Add item for negative caching. + l[localFile] = nil + + file, err := os.Open(localFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "failed to open file %v", localFile) + } + defer file.Close() + + // File present, cache presence. + ids := make([]string, 0) + + var b []byte + reader := bufio.NewReader(file) + for { + b, err = reader.ReadBytes('\n') + if err != nil { + if err != io.EOF { + return errors.Wrapf(err, "failed to read file %v", localFile) + } + break + } + + if bytes.HasPrefix(b, []byte(`#`)) { + ids = append(ids, toHeaderID(b)) + } + } + l[localFile] = &ids + return nil +} + +func toHeaderID(header []byte) string { + var id []byte + + for _, h := range bytes.TrimLeft(bytes.ToLower(header), "#")[1:] { + if (h >= 97 && h <= 122) || (h >= 48 && h <= 57) { + id = append(id, h) + } + switch h { + case '{': + return string(id) + case ' ': + id = append(id, '-') + default: + } + } + return string(id) +} + +// GetAnchorDir returns validated anchor dir against files provided. +func GetAnchorDir(anchorDir string, files []string) (_ string, err error) { + if anchorDir == "" { + anchorDir, err = os.Getwd() + if err != nil { + return "", err + } + } + + // Check if provided files are within anchorDir way. + for _, f := range files { + if !strings.HasPrefix(f, anchorDir) { + return "", errors.Errorf("anchorDir %q is not in path of provided file %q", anchorDir, f) + } + } + return anchorDir, nil +} diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go new file mode 100644 index 0000000..35e459c --- /dev/null +++ b/pkg/mdformatter/linktransformer/link_test.go @@ -0,0 +1,235 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package linktransformer + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/bwplotka/mdox/pkg/mdformatter" + "github.com/bwplotka/mdox/pkg/testutil" + "github.com/go-kit/kit/log" +) + +const ( + testDocWithLinks = `[1](http://myproject.example.com/not-docs.md) [2](.) + +# Yolo + +[3](#yolo) + +# Yolo 2 + +[4](http://myproject.example.com/tip/doc2.md) [5](http://myproject.example.com/v0.15.0/doc2.md) [6](http://not.myproject.example.com/tip/doc2.md) + +[7](http://not.myproject.example.com/tip/a/doc.md#yolo) [8](http://myproject.example.com/tip/a/doc.md) [9](http://not.myproject.example.com/tip/doc2.md#yolo-2) + +[10](http://myproject.example.com/tip/a/does_not_exists_file.md) [11](https://myproject.example.com/tip/a/does_not_exists_file2) [12](http://myproject.example.com/tip/does_not_exists/does_not_exists_dir.md) + +[11](/doc2.md) [12](/a/doc.md#yolo) [13](../doc2.md) [14](../a/../a/../a/../a/doc.md) [15](doc.md) +` +) + +func TestLocalizer_TransformDestination(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test-localizer") + testutil.Ok(t, err) + t.Cleanup(func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }) + + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, "repo", "docs", "a"), os.ModePerm)) + testutil.Ok(t, ioutil.WriteFile(filepath.Join(tmpDir, "repo", "docs", "a", "doc.md"), []byte(testDocWithLinks), os.ModePerm)) + testutil.Ok(t, ioutil.WriteFile(filepath.Join(tmpDir, "repo", "docs", "doc2.md"), []byte(testDocWithLinks), os.ModePerm)) + + logger := log.NewLogfmtLogger(os.Stderr) + anchorDir := filepath.Join(tmpDir, "repo", "docs") + t.Run("no link check, just formatting check should pass.", func(t *testing.T) { + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{ + filepath.Join(tmpDir, "repo", "docs", "a", "doc.md"), + filepath.Join(tmpDir, "repo", "docs", "doc2.md"), + }) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + }) + + t.Run("no domain specified", func(t *testing.T) { + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + NewLocalizer(logger, regexp.MustCompile(`^$`), anchorDir), + )) + testutil.Ok(t, err) + testutil.Equals(t, 1, len(diff), diff.String()) + testutil.Equals(t, fmt.Sprintf(`--- %s/repo/docs/a/doc.md ++++ %s/repo/docs/a/doc.md (formatted) +@@ -11,1 +11,1 @@ + + [10](http://myproject.example.com/tip/a/does_not_exists_file.md) [11](https://myproject.example.com/tip/a/does_not_exists_file2) [12](http://myproject.example.com/tip/does_not_exists/does_not_exists_dir.md) + +-[11](/doc2.md) [12](/a/doc.md#yolo) [13](../doc2.md) [14](../a/../a/../a/../a/doc.md) [15](doc.md) ++[11](../doc2.md) [12](#yolo) [13](../doc2.md) [14](.) [15](.) + +`, tmpDir, tmpDir), diff.String()) + }) + + t.Run("domain specified, but without full path", func(t *testing.T) { + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + NewLocalizer(logger, regexp.MustCompile(`myproject.example.com`), anchorDir), + )) + testutil.Ok(t, err) + testutil.Equals(t, 1, len(diff), diff.String()) + testutil.Equals(t, fmt.Sprintf(`--- %s/repo/docs/a/doc.md ++++ %s/repo/docs/a/doc.md (formatted) +@@ -11,1 +11,1 @@ + + [10](http://myproject.example.com/tip/a/does_not_exists_file.md) [11](https://myproject.example.com/tip/a/does_not_exists_file2) [12](http://myproject.example.com/tip/does_not_exists/does_not_exists_dir.md) + +-[11](/doc2.md) [12](/a/doc.md#yolo) [13](../doc2.md) [14](../a/../a/../a/../a/doc.md) [15](doc.md) ++[11](../doc2.md) [12](#yolo) [13](../doc2.md) [14](.) [15](.) + +`, tmpDir, tmpDir), diff.String()) + }) + + t.Run("domain specified", func(t *testing.T) { + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{filepath.Join(tmpDir, "repo", "docs", "a", "doc.md")}, mdformatter.WithLinkTransformer( + NewLocalizer(logger, regexp.MustCompile(`myproject.example.com/tip/`), anchorDir), + )) + testutil.Ok(t, err) + testutil.Equals(t, 1, len(diff), diff.String()) + testutil.Equals(t, fmt.Sprintf(`--- %s/repo/docs/a/doc.md ++++ %s/repo/docs/a/doc.md (formatted) +@@ -5,3 +5,3 @@ + + # Yolo 2 + +-[4](http://myproject.example.com/tip/doc2.md) [5](http://myproject.example.com/v0.15.0/doc2.md) [6](http://not.myproject.example.com/tip/doc2.md) ++[4](../doc2.md) [5](http://myproject.example.com/v0.15.0/doc2.md) [6](../doc2.md) + +-[7](http://not.myproject.example.com/tip/a/doc.md#yolo) [8](http://myproject.example.com/tip/a/doc.md) [9](http://not.myproject.example.com/tip/doc2.md#yolo-2) ++[7](#yolo) [8](.) [9](../doc2.md#yolo-2) + + [10](http://myproject.example.com/tip/a/does_not_exists_file.md) [11](https://myproject.example.com/tip/a/does_not_exists_file2) [12](http://myproject.example.com/tip/does_not_exists/does_not_exists_dir.md) + +-[11](/doc2.md) [12](/a/doc.md#yolo) [13](../doc2.md) [14](../a/../a/../a/../a/doc.md) [15](doc.md) ++[11](../doc2.md) [12](#yolo) [13](../doc2.md) [14](.) [15](.) + +`, tmpDir, tmpDir), diff.String()) + }) +} + +func TestValidator_TransformDestination(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "test-validator") + testutil.Ok(t, err) + t.Cleanup(func() { testutil.Ok(t, os.RemoveAll(tmpDir)) }) + + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, "repo", "docs", "a"), os.ModePerm)) + testutil.Ok(t, os.MkdirAll(filepath.Join(tmpDir, "repo", "docs", "test"), os.ModePerm)) + testutil.Ok(t, ioutil.WriteFile(filepath.Join(tmpDir, "repo", "docs", "a", "doc.md"), []byte(testDocWithLinks), os.ModePerm)) + testutil.Ok(t, ioutil.WriteFile(filepath.Join(tmpDir, "repo", "docs", "doc2.md"), []byte(testDocWithLinks), os.ModePerm)) + + logger := log.NewLogfmtLogger(os.Stderr) + anchorDir := filepath.Join(tmpDir, "repo", "docs") + t.Run("check valid link", func(t *testing.T) { + testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-link.md") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm)) + + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( + NewValidator(logger, regexp.MustCompile(`^$`), anchorDir), + )) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + }) + + t.Run("check valid local links", func(t *testing.T) { + testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-local-links.md") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte(`# yolo + +[1](.) [2](#yolo) [3](../test/valid-local-links.md) [4](../test/valid-local-links.md#yolo) [5](../a/doc.md) +`), os.ModePerm)) + + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + + diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( + NewValidator(logger, regexp.MustCompile(`^$`), anchorDir), + )) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + }) + + t.Run("check invalid local links", func(t *testing.T) { + testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-local-links.md") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte(`# yolo + +[1](.) [2](#not-yolo) [3](../test2/invalid-local-links.md) [4](../test/invalid-local-links.md#not-yolo) [5](../test/doc.md) +`), os.ModePerm)) + + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( + NewValidator(logger, regexp.MustCompile(`^$`), anchorDir), + )) + testutil.NotOk(t, err) + testutil.Equals(t, "found 4 problems with links.", err.Error()) + }) + + t.Run("check 404 link", func(t *testing.T) { + testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-link.md") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/does-not-exists\n"), os.ModePerm)) + + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( + NewValidator(logger, regexp.MustCompile(`^$`), anchorDir), + )) + testutil.NotOk(t, err) + testutil.Equals(t, "found 1 problems with links.", err.Error()) + }) + + t.Run("check 404 link, ignored", func(t *testing.T) { + testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-link2.md") + testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/does-not-exists\n"), os.ModePerm)) + + diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff), diff.String()) + + _, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer( + NewValidator(logger, regexp.MustCompile(`://bwplotka.dev`), anchorDir), + )) + testutil.Ok(t, err) + }) +} + +func TestGetAnchorDir(t *testing.T) { + pwd, err := os.Getwd() + testutil.Ok(t, err) + + // Consider parametrizing this. + anchorDir, err := GetAnchorDir("", []string{}) + testutil.Ok(t, err) + testutil.Equals(t, pwd, anchorDir) + + anchorDir, err = GetAnchorDir("/root", []string{}) + testutil.Ok(t, err) + testutil.Equals(t, "/root", anchorDir) + + _, err = GetAnchorDir("/root", []string{"/root/something.md", "/home/something/file.md", "/root/a/b/c/file.md"}) + testutil.NotOk(t, err) + testutil.Equals(t, "anchorDir \"/root\" is not in path of provided file \"/home/something/file.md\"", err.Error()) + + _, err = GetAnchorDir("/root", []string{"/root/something.md", "/root/something/file.md", "/root/a/b/c/file.md"}) + testutil.Ok(t, err) + testutil.Equals(t, "/root", anchorDir) +} diff --git a/pkg/mdformatter/mdformatter.go b/pkg/mdformatter/mdformatter.go index edce2b6..b955bc1 100644 --- a/pkg/mdformatter/mdformatter.go +++ b/pkg/mdformatter/mdformatter.go @@ -11,6 +11,8 @@ import ( "os" "github.com/Kunde21/markdownfmt/v2/markdown" + "github.com/bwplotka/mdox/pkg/gitdiff" + "github.com/bwplotka/mdox/pkg/merrors" "github.com/bwplotka/mdox/pkg/runutil" "github.com/go-kit/kit/log" "github.com/gohugoio/hugo/parser/pageparser" @@ -22,14 +24,17 @@ import ( type FrontMatterTransformer interface { TransformFrontMatter(ctx context.Context, docPath string, frontMatter map[string]interface{}) ([]byte, error) + io.Closer } type LinkTransformer interface { TransformDestination(ctx context.Context, docPath string, destination []byte) ([]byte, error) + io.Closer } type CodeBlockTransformer interface { TransformCodeBlock(ctx context.Context, docPath string, infoString []byte, code []byte) ([]byte, error) + io.Closer } type Formatter struct { @@ -84,17 +89,21 @@ func (RemoveFrontMatter) TransformFrontMatter(_ context.Context, _ string, front return nil, nil } -// Format formats given markdown files in-place. Check `With...` function to see what modifiers you can add. +func (RemoveFrontMatter) Close() error { return nil } + +// Format formats given markdown files in-place. IsFormatted `With...` function to see what modifiers you can add. func Format(ctx context.Context, logger log.Logger, files []string, opts ...Option) error { f := New(ctx, opts...) b := bytes.Buffer{} // TODO(bwplotka): Do Concurrently. + + errs := merrors.New() for _, fn := range files { - if err := func() error { + errs.Add(func() error { file, err := os.OpenFile(fn, os.O_RDWR, 0) if err != nil { - return errors.Wrapf(err, "read %v", fn) + return errors.Wrapf(err, "open %v", fn) } defer runutil.CloseWithLogOnErr(logger, file, "close file %v", fn) @@ -105,14 +114,65 @@ func Format(ctx context.Context, logger log.Logger, files []string, opts ...Opti n, err := file.WriteAt(b.Bytes(), 0) if err != nil { - return errors.Wrapf(err, "write: %v", fn) + return errors.Wrapf(err, "write %v", fn) } return file.Truncate(int64(n)) - }(); err != nil { - return err - } + }()) } - return nil + return errs.Err() +} + +type Diffs []gitdiff.Diff + +func (d Diffs) String() string { + if len(d) == 0 { + return "files the same; no diff" + } + + b := bytes.Buffer{} + for _, diff := range d { + _, _ = b.Write(diff.ToCombinedFormat()) + } + return b.String() +} + +// IsFormatted tries to formats given markdown files and return Diff if files are not formatted. +// If diff is empty it means all files are formatted. +func IsFormatted(ctx context.Context, logger log.Logger, files []string, opts ...Option) (diffs Diffs, err error) { + f := New(ctx, opts...) + b := bytes.Buffer{} + + // TODO(bwplotka): Do Concurrently. + errs := merrors.New() + for _, fn := range files { + errs.Add(func() error { + file, err := os.OpenFile(fn, os.O_RDWR, 0) + if err != nil { + return errors.Wrapf(err, "open %v", fn) + } + defer runutil.CloseWithLogOnErr(logger, file, "close file %v", fn) + + b.Reset() + if err := f.Format(file, &b); err != nil { + return err + } + + if _, err := file.Seek(0, 0); err != nil { + return err + } + + in, err := ioutil.ReadAll(file) + if err != nil { + return errors.Wrapf(err, "read all %v", fn) + } + + if !bytes.Equal(in, b.Bytes()) { + diffs = append(diffs, gitdiff.CompareBytes(in, fn, b.Bytes(), fn+" (formatted)")) + } + return nil + }()) + } + return diffs, errs.Err() } // Format writes formatted input file into out writer. @@ -122,7 +182,7 @@ func (f *Formatter) Format(file *os.File, out io.Writer) error { f: f, docPath: file.Name(), } - gm := goldmark.New( + gm1 := goldmark.New( goldmark.WithExtensions( extension.GFM, ), @@ -134,6 +194,18 @@ func (f *Formatter) Format(file *os.File, out io.Writer) error { goldmark.WithRenderer(nopOpsRenderer{Renderer: t}), ) + gm2 := goldmark.New( + goldmark.WithExtensions( + extension.GFM, + ), + goldmark.WithParserOptions( + parser.WithAttribute(), // Enable # headers {#custom-ids}. + parser.WithHeadingAttribute(), + ), + goldmark.WithParserOptions(), + goldmark.WithRenderer(markdown.NewRenderer()), + ) + b, err := ioutil.ReadAll(file) if err != nil { return errors.Wrapf(err, "read %v", file.Name()) @@ -154,17 +226,22 @@ func (f *Formatter) Format(file *os.File, out io.Writer) error { if _, err := out.Write(hdr); err != nil { return err } + if err := f.fm.Close(); err != nil { + return err + } } // Hack: run Convert two times to ensure deterministic whitespace alignment. // This also immediately show transformers which are not working well together etc. tmp := bytes.Buffer{} - if err := gm.Convert(content, &tmp); err != nil { - return errors.Wrapf(err, "Formatter %v", f) + if err := gm1.Convert(content, &tmp); err != nil { + return errors.Wrapf(err, "first formatting phase for %v", file.Name()) } - - if err := gm.Convert(tmp.Bytes(), out); err != nil { - return errors.Wrapf(err, "Formatter %v", f) + if err := t.Close(); err != nil { + return err + } + if err := gm2.Convert(tmp.Bytes(), out); err != nil { + return errors.Wrapf(err, "second formatting phase for %v", file.Name()) } return nil } diff --git a/pkg/mdformatter/mdformatter_test.go b/pkg/mdformatter/mdformatter_test.go index 88ba527..e86bedc 100644 --- a/pkg/mdformatter/mdformatter_test.go +++ b/pkg/mdformatter/mdformatter_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/bwplotka/mdox/pkg/testutil" + "github.com/go-kit/kit/log" ) func TestFormat_FormatSingle_NoTransformers(t *testing.T) { @@ -40,9 +41,25 @@ func TestFormat_FormatSingle_NoTransformers(t *testing.T) { }) } -type mockLinkTransformer struct{} +func TestCheck_NoTransformers(t *testing.T) { + diff, err := IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/formatted.md"}) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(diff)) + testutil.Equals(t, "files the same; no diff", diff.String()) + + diff, err = IsFormatted(context.Background(), log.NewNopLogger(), []string{"testdata/not_formatted.md"}) + testutil.Ok(t, err) + + exp, err := ioutil.ReadFile("testdata/not_formatted.md.diff") + testutil.Ok(t, err) + testutil.Equals(t, string(exp), diff.String()) +} -func (mockLinkTransformer) TransformDestination(_ context.Context, docPath string, destination []byte) ([]byte, error) { +type mockLinkTransformer struct { + closed bool +} + +func (*mockLinkTransformer) TransformDestination(_ context.Context, docPath string, destination []byte) ([]byte, error) { if bytes.HasPrefix(destination, []byte("$$-")) { return destination, nil } @@ -54,13 +71,19 @@ func (mockLinkTransformer) TransformDestination(_ context.Context, docPath strin return b.Bytes(), nil } +func (m *mockLinkTransformer) Close() error { + m.closed = true + return nil +} + func TestFormat_FormatSingle_Transformers(t *testing.T) { file, err := os.OpenFile("testdata/not_formatted.md", os.O_RDONLY, 0) testutil.Ok(t, err) defer file.Close() + m := &mockLinkTransformer{} f := New(context.Background()) - f.link = mockLinkTransformer{} + f.link = m exp, err := ioutil.ReadFile("testdata/formatted_and_transformed.md") testutil.Ok(t, err) @@ -80,4 +103,6 @@ func TestFormat_FormatSingle_Transformers(t *testing.T) { testutil.Ok(t, f.Format(file2, &buf)) testutil.Equals(t, string(exp), buf.String()) }) + + testutil.Equals(t, true, m.closed) } diff --git a/pkg/mdgen/mdgen.go b/pkg/mdformatter/mdgen/mdgen.go similarity index 95% rename from pkg/mdgen/mdgen.go rename to pkg/mdformatter/mdgen/mdgen.go index f87110c..f66f829 100644 --- a/pkg/mdgen/mdgen.go +++ b/pkg/mdformatter/mdgen/mdgen.go @@ -26,7 +26,7 @@ func NewCodeBlockTransformer() *genCodeBlockTransformer { return &genCodeBlockTransformer{} } -func (t *genCodeBlockTransformer) TransformCodeBlock(ctx context.Context, docPath string, infoString []byte, code []byte) ([]byte, error) { +func (t *genCodeBlockTransformer) TransformCodeBlock(ctx context.Context, _ string, infoString []byte, code []byte) ([]byte, error) { if len(infoString) == 0 { return code, nil } @@ -89,6 +89,8 @@ func (t *genCodeBlockTransformer) TransformCodeBlock(ctx context.Context, docPat panic("should never get here") } +func (t *genCodeBlockTransformer) Close() error { return nil } + func genGo(ctx context.Context, moduleRoot string, typePath string) ([]byte, error) { // TODO(bwplotka): To be done. return nil, nil diff --git a/pkg/mdgen/mdgen_test.go b/pkg/mdformatter/mdgen/mdgen_test.go similarity index 100% rename from pkg/mdgen/mdgen_test.go rename to pkg/mdformatter/mdgen/mdgen_test.go diff --git a/pkg/mdgen/testdata/cfg.go b/pkg/mdformatter/mdgen/testdata/cfg.go similarity index 100% rename from pkg/mdgen/testdata/cfg.go rename to pkg/mdformatter/mdgen/testdata/cfg.go diff --git a/pkg/mdgen/testdata/mdgen_formatted.md b/pkg/mdformatter/mdgen/testdata/mdgen_formatted.md similarity index 100% rename from pkg/mdgen/testdata/mdgen_formatted.md rename to pkg/mdformatter/mdgen/testdata/mdgen_formatted.md diff --git a/pkg/mdgen/testdata/mdgen_not_formatted.md b/pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md similarity index 100% rename from pkg/mdgen/testdata/mdgen_not_formatted.md rename to pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md diff --git a/pkg/mdgen/testdata/out.sh b/pkg/mdformatter/mdgen/testdata/out.sh similarity index 100% rename from pkg/mdgen/testdata/out.sh rename to pkg/mdformatter/mdgen/testdata/out.sh diff --git a/pkg/mdgen/testdata/out2.sh b/pkg/mdformatter/mdgen/testdata/out2.sh similarity index 100% rename from pkg/mdgen/testdata/out2.sh rename to pkg/mdformatter/mdgen/testdata/out2.sh diff --git a/pkg/mdformatter/testdata/formatted.md b/pkg/mdformatter/testdata/formatted.md index 6e7934a..bcd6965 100644 --- a/pkg/mdformatter/testdata/formatted.md +++ b/pkg/mdformatter/testdata/formatted.md @@ -4,6 +4,8 @@ Feel free to check the free, in-browser interactive tutorial [as Katacoda Thanos On top of this feel free to go through our tutorial presented here: +Yolo link https://thanos.io/some Yolo email something@gmail.com + ### Prometheus Thanos is based on Prometheus. With Thanos you use more or less Prometheus features depending on the deployment model, however Prometheus always stays as integral foundation for *collecting metrics* and alerting using local data. diff --git a/pkg/mdformatter/testdata/formatted_and_transformed.md b/pkg/mdformatter/testdata/formatted_and_transformed.md index 72f66be..5632343 100644 --- a/pkg/mdformatter/testdata/formatted_and_transformed.md +++ b/pkg/mdformatter/testdata/formatted_and_transformed.md @@ -4,6 +4,8 @@ Feel free to check the free, in-browser interactive tutorial [as Katacoda Thanos On top of this feel free to go through our tutorial presented here: +Yolo link $$-https://thanos.io/some-testdata/not_formatted.md-$$ Yolo email something@gmail.com + ### Prometheus Thanos is based on Prometheus. With Thanos you use more or less Prometheus features depending on the deployment model, however Prometheus always stays as integral foundation for *collecting metrics* and alerting using local data. diff --git a/pkg/mdformatter/testdata/not_formatted.md b/pkg/mdformatter/testdata/not_formatted.md index 7da541d..500eb19 100644 --- a/pkg/mdformatter/testdata/not_formatted.md +++ b/pkg/mdformatter/testdata/not_formatted.md @@ -13,6 +13,9 @@ We will be progressively updating our Katacoda Course with more scenarios. On top of this feel free to go through our tutorial presented here: +Yolo link https://thanos.io/some +Yolo email something@gmail.com + ### Prometheus Thanos is based on Prometheus. With Thanos you use more or less Prometheus features depending on the deployment model, however diff --git a/pkg/mdformatter/testdata/not_formatted.md.diff b/pkg/mdformatter/testdata/not_formatted.md.diff new file mode 100644 index 0000000..8860021 --- /dev/null +++ b/pkg/mdformatter/testdata/not_formatted.md.diff @@ -0,0 +1,150 @@ +--- testdata/not_formatted.md ++++ testdata/not_formatted.md (formatted) +@@ -0,14 +0,3 @@ +---- +-title: Quick Tutorial +-type: docs +-menu: thanos +-weight: 1 +-slug: /quick-tutorial.md +---- +- + # Quick Tutorial + +-Feel free to check the free, in-browser interactive tutorial [as Katacoda Thanos Course](https://katacoda.com/bwplotka/courses/thanos) +-We will be progressively updating our Katacoda Course with more scenarios. ++Feel free to check the free, in-browser interactive tutorial [as Katacoda Thanos Course](https://katacoda.com/bwplotka/courses/thanos) We will be progressively updating our Katacoda Course with more scenarios. + + On top of this feel free to go through our tutorial presented here: + +-Yolo link https://thanos.io/some +-Yolo email something@gmail.com ++Yolo link https://thanos.io/some Yolo email something@gmail.com + + ### Prometheus + +-Thanos is based on Prometheus. With Thanos you use more or less Prometheus features depending on the deployment model, however +-Prometheus always stays as integral foundation for *collecting metrics* and alerting using local data. ++Thanos is based on Prometheus. With Thanos you use more or less Prometheus features depending on the deployment model, however Prometheus always stays as integral foundation for *collecting metrics* and alerting using local data. + + Thanos bases itself on vanilla [Prometheus](https://prometheus.io/) (v2.2.1+). We plan to support *all* Prometheus version beyond this version. + +@@ -74,4 +63,4 @@ + + If you are not interested in backing up any data, the `--objstore.config-file` flag can simply be omitted. + +-* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ ++* *[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)* +-* _[Example Deploying sidecar using official Prometheus Helm Chart](/tutorials/kubernetes-helm/README.md)_ ++* *[Example Deploying sidecar using official Prometheus Helm Chart](/tutorials/kubernetes-helm/README.md)* +-* _[Details & Config for other object stores](storage.md)_ ++* *[Details & Config for other object stores](storage.md)* + + #### Store API + +-The Sidecar component implements and exposes a gRPC _[Store API](/pkg/store/storepb/rpc.proto#L19)_. The sidecar implementation allows you to query the metric data stored in Prometheus. ++The Sidecar component implements and exposes a gRPC *[Store API](/pkg/store/storepb/rpc.proto#L19)*. The sidecar implementation allows you to query the metric data stored in Prometheus. + + Let's extend the Sidecar in the previous section to connect to a Prometheus server, and expose the Store API. + +@@ -93,4 +82,2 @@ + --grpc-address 0.0.0.0:19090 # GRPC endpoint for StoreAPI + ``` + +-* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ ++* *[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)* + + ### Uploading old metrics. + +-When sidecar is run with the `--shipper.upload-compacted` flag it will sync all older existing blocks from the Prometheus local storage on startup. +-NOTE: This assumes you never run sidecar with block uploading against this bucket. Otherwise manual steps are needed to remove overlapping blocks from the bucket. +-Those will be suggested by the sidecar verification process. ++When sidecar is run with the `--shipper.upload-compacted` flag it will sync all older existing blocks from the Prometheus local storage on startup. NOTE: This assumes you never run sidecar with block uploading against this bucket. Otherwise manual steps are needed to remove overlapping blocks from the bucket. Those will be suggested by the sidecar verification process. + + #### External Labels + +@@ -165,5 +152,3 @@ + + Go to the configured HTTP address, and you should now be able to query across all Prometheus instances and receive de-duplicated data. + +-* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-query-deployment.yaml)_ ++* *[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-query-deployment.yaml)* + + #### Communication Between Components + +-The only required communication between nodes is for Thanos Querier to be able to reach gRPC storeAPIs you provide. Thanos Querier periodically calls Info endpoint to collect up-to-date metadata as well as checking the health of given StoreAPI. +-The metadata includes the information about time windows and external labels for each node. ++The only required communication between nodes is for Thanos Querier to be able to reach gRPC storeAPIs you provide. Thanos Querier periodically calls Info endpoint to collect up-to-date metadata as well as checking the health of given StoreAPI. The metadata includes the information about time windows and external labels for each node. + +-There are various ways to tell query component about the StoreAPIs it should query data from. The simplest way is to use a static list of well known addresses to query. +-These are repeatable so can add as many endpoint as needed. You can put DNS domain prefixed by `dns+` or `dnssrv+` to have Thanos Query do an `A` or `SRV` lookup to get all required IPs to communicate with. ++There are various ways to tell query component about the StoreAPIs it should query data from. The simplest way is to use a static list of well known addresses to query. These are repeatable so can add as many endpoint as needed. You can put DNS domain prefixed by `dns+` or `dnssrv+` to have Thanos Query do an `A` or `SRV` lookup to get all required IPs to communicate with. + + ```bash + thanos query \ +@@ -186,4 +171,2 @@ + + Read more details [here](service-discovery.md). + +-* _[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)_ ++* *[Example Kubernetes manifests using Prometheus operator](https://github.com/coreos/prometheus-operator/tree/master/example/thanos)* + + ### [Store Gateway](components/store.md) + +-As the sidecar backs up data into the object storage of your choice, you can decrease Prometheus retention and store less locally. However we need a way to query all that historical data again. +-The store gateway does just that by implementing the same gRPC data API as the sidecars but backing it with data it can find in your object storage bucket. +-Just like sidecars and query nodes, the store gateway exposes StoreAPI and needs to be discovered by Thanos Querier. ++As the sidecar backs up data into the object storage of your choice, you can decrease Prometheus retention and store less locally. However we need a way to query all that historical data again. The store gateway does just that by implementing the same gRPC data API as the sidecars but backing it with data it can find in your object storage bucket. Just like sidecars and query nodes, the store gateway exposes StoreAPI and needs to be discovered by Thanos Querier. + + ```bash + thanos store \ +@@ -204,1 +187,1 @@ + + The store gateway occupies small amounts of disk space for caching basic information about data in the object storage. This will rarely exceed more than a few gigabytes and is used to improve restart times. It is useful but not required to preserve it across restarts. + +-* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-store-statefulSet.yaml)_ ++* *[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/manifests/thanos-store-statefulSet.yaml)* + + ### [Compactor](components/compact.md) + +@@ -221,8 +204,4 @@ + + The compactor is not in the critical path of querying or data backup. It can either be run as a periodic batch job or be left running to always compact data as soon as possible. It is recommended to provide 100-300GB of local disk space for data processing. + +-_NOTE: The compactor must be run as a **singleton** and must not run when manually modifying data in the bucket._ ++*NOTE: The compactor must be run as a **singleton** and must not run when manually modifying data in the bucket.* + +-* _[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/examples/all/manifests/thanos-compact-statefulSet.yaml)_ ++* *[Example Kubernetes manifest](https://github.com/thanos-io/kube-thanos/blob/master/examples/all/manifests/thanos-compact-statefulSet.yaml)* + + ### [Ruler/Rule](components/rule.md) + +-In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), +-which does rule and alert evaluation on top of a given Thanos Querier. +- ++In case of Prometheus with Thanos sidecar does not have enough retention, or if you want to have alerts or recording rules that requires global view, Thanos has just the component for that: the [Ruler](components/rule.md), which does rule and alert evaluation on top of a given Thanos Querier. + +- + ## Flags +- +-[embedmd]:# (flags/rule.txt $) ++ + ```$ + usage: thanos rule [] + +@@ -409,1 +388,0 @@ + + The configuration format is the following: + +-[embedmd]:# (../flags/config_rule_alerting.txt yaml) + ```yaml + alertmanagers: + - http_config: +@@ -434,2 +412,1 @@ + path_prefix: "" + timeout: 10s + api_version: v1 +-``` +- ++``` + diff --git a/pkg/mdformatter/transformer.go b/pkg/mdformatter/transformer.go index 0498276..9979570 100644 --- a/pkg/mdformatter/transformer.go +++ b/pkg/mdformatter/transformer.go @@ -4,8 +4,10 @@ package mdformatter import ( + "bytes" "io" + "github.com/bwplotka/mdox/pkg/merrors" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/renderer" "github.com/yuin/goldmark/text" @@ -37,28 +39,38 @@ func (t *transformer) Render(w io.Writer, source []byte, node ast.Node) error { var err error switch typedNode := n.(type) { case *ast.Link: - if t.f.link == nil { + if !entering || t.f.link == nil { return ast.WalkSkipChildren, nil } - if entering { - typedNode.Destination, err = t.f.link.TransformDestination(t.f.ctx, t.docPath, typedNode.Destination) - if err != nil { - return ast.WalkStop, err - } + typedNode.Destination, err = t.f.link.TransformDestination(t.f.ctx, t.docPath, typedNode.Destination) + if err != nil { + return ast.WalkStop, err } + case *ast.AutoLink: + if !entering || t.f.link == nil || typedNode.AutoLinkType != ast.AutoLinkURL { + return ast.WalkSkipChildren, nil + } + dest, err := t.f.link.TransformDestination(t.f.ctx, t.docPath, typedNode.URL(source)) + if err != nil { + return ast.WalkStop, err + } + if bytes.Equal(dest, typedNode.URL(source)) { + return ast.WalkSkipChildren, nil + } + repl := ast.NewString(dest) + repl.SetParent(n) + n.Parent().ReplaceChild(n.Parent(), n, repl) case *ast.FencedCodeBlock: - if t.f.cb == nil || typedNode.Info == nil { + if !entering || t.f.cb == nil || typedNode.Info == nil { return ast.WalkSkipChildren, nil } - if entering { - blockContent, err := t.f.cb.TransformCodeBlock(t.f.ctx, t.docPath, typedNode.Info.Text(source), typedNode.Text(source)) - if err != nil { - return ast.WalkStop, err - } - if blockContent != nil { - replaceContent(&typedNode.BaseBlock, len(source), blockContent) - source = append(source, blockContent...) - } + blockContent, err := t.f.cb.TransformCodeBlock(t.f.ctx, t.docPath, typedNode.Info.Text(source), typedNode.Text(source)) + if err != nil { + return ast.WalkStop, err + } + if blockContent != nil { + replaceContent(&typedNode.BaseBlock, len(source), blockContent) + source = append(source, blockContent...) } default: return ast.WalkContinue, nil @@ -70,6 +82,17 @@ func (t *transformer) Render(w io.Writer, source []byte, node ast.Node) error { return t.wrapped.Render(w, source, node) } +func (t *transformer) Close() error { + errs := merrors.New() + if t.f.link != nil { + errs.Add(t.f.link.Close()) + } + if t.f.cb != nil { + errs.Add(t.f.cb.Close()) + } + return errs.Err() +} + func replaceContent(b *ast.BaseBlock, lastSegmentStop int, content []byte) { s := text.NewSegments() // NOTE(bwplotka): This feels like hack, because we pack all lines in single line. But it works (: diff --git a/pkg/merrors/errors.go b/pkg/merrors/errors.go new file mode 100644 index 0000000..a6f7d5a --- /dev/null +++ b/pkg/merrors/errors.go @@ -0,0 +1,172 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package merrors + +import ( + "bytes" + stderrors "errors" + "fmt" +) + +// NilOrMultiError type allows combining multiple errors into one. +type NilOrMultiError struct { + errs []error +} + +// New returns NilOrMultiError with provided errors added if not nil. +func New(errs ...error) *NilOrMultiError { + m := &NilOrMultiError{} + m.Add(errs...) + return m +} + +// Add adds single or many errors to the error list. Each error is added only if not nil. +// If the error is a multiError type, the errors inside multiError are added to the main NilOrMultiError. +func (e *NilOrMultiError) Add(errs ...error) { + for _, err := range errs { + if err == nil { + continue + } + if merr, ok := err.(multiError); ok { + e.errs = append(e.errs, merr.errs...) + continue + } + e.errs = append(e.errs, err) + } +} + +// Err returns the error list as an Result (also implements error) or nil if it is empty. +func (e NilOrMultiError) Err() Result { + if len(e.errs) == 0 { + return nil + } + return multiError(e) +} + +// Result is extended error interface that allows to use returned read-only multi error in more advanced ways. +type Result interface { + error + + // Errors returns underlying errors. + Errors() []error + + // As finds the first error in multiError slice of error chains that matches target, and if so, sets + // target to that error value and returns true. Otherwise, it returns false. + // + // An error matches target if the error's concrete value is assignable to the value + // pointed to by target, or if the error has a method As(interface{}) bool such that + // As(target) returns true. In the latter case, the As method is responsible for + // setting target. + As(target interface{}) bool + // Is returns true if any error in multiError's slice of error chains matches the given target or + // if the target is of multiError type. + // + // An error is considered to match a target if it is equal to that target or if + // it implements a method Is(error) bool such that Is(target) returns true. + Is(target error) bool + // Count returns the number of multi error' errors that match the given target. + // Matching is defined as in Is method. + Count(target error) int +} + +// multiError implements the error and Result interfaces, and it represents NilOrMultiError (in other words []error) with at least one error inside it. +// NOTE: This type is useful to make sure that NilOrMultiError is not accidentally used for err != nil check. +type multiError struct { + errs []error +} + +// Errors returns underlying errors. +func (e multiError) Errors() []error { + return e.errs +} + +// Error returns a concatenated string of the contained errors. +func (e multiError) Error() string { + var buf bytes.Buffer + + if len(e.errs) > 1 { + fmt.Fprintf(&buf, "%d errors: ", len(e.errs)) + } + + for i, err := range e.errs { + if i != 0 { + buf.WriteString("; ") + } + buf.WriteString(err.Error()) + } + + return buf.String() +} + +// As finds the first error in multiError slice of error chains that matches target, and if so, sets +// target to that error value and returns true. Otherwise, it returns false. +// +// An error matches target if the error's concrete value is assignable to the value +// pointed to by target, or if the error has a method As(interface{}) bool such that +// As(target) returns true. In the latter case, the As method is responsible for +// setting target. +func (e multiError) As(target interface{}) bool { + if t, ok := target.(*multiError); ok { + *t = e + return true + } + + for _, err := range e.errs { + if stderrors.As(err, target) { + return true + } + } + return false +} + +// Is returns true if any error in multiError's slice of error chains matches the given target or +// if the target is of multiError type. +// +// An error is considered to match a target if it is equal to that target or if +// it implements a method Is(error) bool such that Is(target) returns true. +func (e multiError) Is(target error) bool { + if m, ok := target.(multiError); ok { + if len(m.errs) != len(e.errs) { + return false + } + for i := 0; i < len(e.errs); i++ { + if !stderrors.Is(m.errs[i], e.errs[i]) { + return false + } + } + return true + } + for _, err := range e.errs { + if stderrors.Is(err, target) { + return true + } + } + return false +} + +// Count returns the number of all multi error' errors that match the given target (including nested multi errors). +// Matching is defined as in Is method. +func (e multiError) Count(target error) (count int) { + for _, err := range e.errs { + if inner, ok := AsMulti(err); ok { + count += inner.Count(target) + continue + } + + if stderrors.Is(err, target) { + count++ + } + } + return count +} + +// As casts error to multi error read only interface. It returns multi error and true if error matches multi error as +// defined by As method. If returns false if no multi error can be found. +func AsMulti(err error) (Result, bool) { + m := multiError{} + if !stderrors.As(err, &m) { + return nil, false + } + return m, true +} diff --git a/pkg/merrors/errors_test.go b/pkg/merrors/errors_test.go new file mode 100644 index 0000000..b11460d --- /dev/null +++ b/pkg/merrors/errors_test.go @@ -0,0 +1,223 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package merrors_test + +import ( + stderrors "errors" + "testing" + + "github.com/bwplotka/mdox/pkg/merrors" + pkgerrors "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestNilMultiError(t *testing.T) { + require.NoError(t, merrors.New().Err()) + require.NoError(t, merrors.New(nil, nil, nil).Err()) + + e := merrors.New() + e.Add() + require.NoError(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add() + require.NoError(t, e.Err()) + + e = merrors.New() + e.Add(nil, nil, nil) + require.NoError(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(nil, nil, nil) + require.NoError(t, e.Err()) +} + +func TestMultiError(t *testing.T) { + err := stderrors.New("test1") + require.Error(t, merrors.New(err).Err()) + require.Error(t, merrors.New(nil, err, nil).Err()) + + e := merrors.New(err) + e.Add() + require.Error(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(err) + require.Error(t, e.Err()) + + e = merrors.New(err) + e.Add(nil, nil, nil) + require.Error(t, e.Err()) + + e = merrors.New(nil, nil, nil) + e.Add(nil, err, nil) + require.Error(t, e.Err()) + + require.Error(t, func() error { + return e.Err() + }()) + + require.NoError(t, func() error { + return merrors.New(nil, nil, nil).Err() + }()) +} + +func TestMultiError_Error(t *testing.T) { + err := stderrors.New("test1") + + require.Equal(t, "test1", merrors.New(err).Err().Error()) + require.Equal(t, "test1", merrors.New(err, nil).Err().Error()) + require.Equal(t, "4 errors: test1; test1; test2; test3", merrors.New(err, err, stderrors.New("test2"), nil, stderrors.New("test3")).Err().Error()) +} + +type customErr struct{ error } + +type customErr2 struct{ error } + +type customErr3 struct{ error } + +func TestMultiError_As(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + + require.True(t, stderrors.As(err, &err)) + require.True(t, stderrors.As(err, &customErr{})) + + require.False(t, stderrors.As(err, &customErr2{})) + require.False(t, stderrors.As(err, &customErr3{})) + + // This is just to show limitation of std As. + require.False(t, stderrors.As(&err, &err)) + require.False(t, stderrors.As(&err, &customErr{})) + require.False(t, stderrors.As(&err, &customErr2{})) + require.False(t, stderrors.As(&err, &customErr3{})) + + e := merrors.New(err).Err() + require.True(t, stderrors.As(e, &customErr{})) + same := merrors.New(err).Err() + require.True(t, stderrors.As(e, &same)) + require.False(t, stderrors.As(e, &customErr2{})) + require.False(t, stderrors.As(e, &customErr3{})) + + e2 := merrors.New(err, customErr3{error: stderrors.New("some")}).Err() + require.True(t, stderrors.As(e2, &customErr{})) + require.True(t, stderrors.As(e2, &customErr3{})) + require.False(t, stderrors.As(e2, &customErr2{})) + + // Wrapped. + e3 := pkgerrors.Wrap(merrors.New(err, customErr3{}).Err(), "wrap") + require.True(t, stderrors.As(e3, &customErr{})) + require.True(t, stderrors.As(e3, &customErr3{})) + require.False(t, stderrors.As(e3, &customErr2{})) + + // This is just to show limitation of std As. + e4 := merrors.New(err, &customErr3{}).Err() + require.False(t, stderrors.As(e4, &customErr2{})) + require.False(t, stderrors.As(e4, &customErr3{})) +} + +func TestMultiError_Is(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + + require.True(t, stderrors.Is(err, err)) + require.True(t, stderrors.Is(err, customErr{error: err.error})) + require.False(t, stderrors.Is(err, &err)) + require.False(t, stderrors.Is(err, customErr{})) + require.False(t, stderrors.Is(err, customErr{error: stderrors.New("err1")})) + require.False(t, stderrors.Is(err, customErr2{})) + require.False(t, stderrors.Is(err, customErr3{})) + + require.True(t, stderrors.Is(&err, &err)) + require.False(t, stderrors.Is(&err, &customErr{error: err.error})) + require.False(t, stderrors.Is(&err, &customErr2{})) + require.False(t, stderrors.Is(&err, &customErr3{})) + + e := merrors.New(err).Err() + require.True(t, stderrors.Is(e, err)) + require.True(t, stderrors.Is(err, customErr{error: err.error})) + require.True(t, stderrors.Is(e, e)) + require.True(t, stderrors.Is(e, merrors.New(err).Err())) + require.False(t, stderrors.Is(e, &err)) + require.False(t, stderrors.Is(err, customErr{})) + require.False(t, stderrors.Is(e, customErr2{})) + require.False(t, stderrors.Is(e, customErr3{})) + + e2 := merrors.New(err, customErr3{}).Err() + require.True(t, stderrors.Is(e2, err)) + require.True(t, stderrors.Is(e2, customErr3{})) + require.True(t, stderrors.Is(e2, merrors.New(err, customErr3{}).Err())) + require.False(t, stderrors.Is(e2, merrors.New(customErr3{}, err).Err())) + require.False(t, stderrors.Is(e2, customErr{})) + require.False(t, stderrors.Is(e2, customErr2{})) + + // Wrapped. + e3 := pkgerrors.Wrap(merrors.New(err, customErr3{}).Err(), "wrap") + require.True(t, stderrors.Is(e3, err)) + require.True(t, stderrors.Is(e3, customErr3{})) + require.False(t, stderrors.Is(e3, customErr{})) + require.False(t, stderrors.Is(e3, customErr2{})) + + exact := &customErr3{} + e4 := merrors.New(err, exact).Err() + require.True(t, stderrors.Is(e4, err)) + require.True(t, stderrors.Is(e4, exact)) + require.True(t, stderrors.Is(e4, merrors.New(err, exact).Err())) + require.False(t, stderrors.Is(e4, customErr{})) + require.False(t, stderrors.Is(e4, customErr2{})) + require.False(t, stderrors.Is(e4, &customErr3{})) +} + +func TestMultiError_Count(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + merr := merrors.New() + merr.Add(customErr3{}) + + m, ok := merrors.AsMulti(merr.Err()) + require.True(t, ok) + require.Equal(t, 0, m.Count(err)) + require.Equal(t, 1, m.Count(customErr3{})) + + merr.Add(customErr3{}) + merr.Add(customErr3{}) + + m, ok = merrors.AsMulti(merr.Err()) + require.True(t, ok) + require.Equal(t, 0, m.Count(err)) + require.Equal(t, 3, m.Count(customErr3{})) + + // Nest multi errors with wraps. + merr2 := merrors.New() + merr2.Add(customErr3{}) + merr2.Add(customErr3{}) + merr2.Add(customErr3{}) + + merr3 := merrors.New() + merr3.Add(customErr3{}) + merr3.Add(customErr3{}) + + // Wrap it so Add cannot add inner errors in. + merr2.Add(pkgerrors.Wrap(merr3.Err(), "wrap")) + merr.Add(pkgerrors.Wrap(merr2.Err(), "wrap")) + + m, ok = merrors.AsMulti(merr.Err()) + require.True(t, ok) + require.Equal(t, 0, m.Count(err)) + require.Equal(t, 8, m.Count(customErr3{})) +} + +func TestAsMulti(t *testing.T) { + err := customErr{error: stderrors.New("err1")} + merr := merrors.New(err, customErr3{}).Err() + wrapped := pkgerrors.Wrap(merr, "wrap") + + _, ok := merrors.AsMulti(err) + require.False(t, ok) + + m, ok := merrors.AsMulti(merr) + require.True(t, ok) + require.True(t, stderrors.Is(m, merr)) + + m, ok = merrors.AsMulti(wrapped) + require.True(t, ok) + require.True(t, stderrors.Is(m, merr)) +} diff --git a/pkg/runutil/runutil.go b/pkg/runutil/runutil.go index 3cd7495..3fca430 100644 --- a/pkg/runutil/runutil.go +++ b/pkg/runutil/runutil.go @@ -55,13 +55,13 @@ package runutil import ( - "bytes" "fmt" "io" "io/ioutil" "os" "time" + "github.com/bwplotka/mdox/pkg/merrors" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" @@ -141,12 +141,7 @@ func ExhaustCloseWithLogOnErr(logger log.Logger, r io.ReadCloser, format string, // CloseWithErrCapture runs function and on error return error by argument including the given error (usually // from caller function). func CloseWithErrCapture(err *error, closer io.Closer, format string, a ...interface{}) { - merr := MultiError{} - - merr.Add(*err) - merr.Add(errors.Wrapf(closer.Close(), format, a...)) - - *err = merr.Err() + *err = merrors.New(*err, errors.Wrapf(closer.Close(), format, a...)).Err() } // ExhaustCloseWithErrCapture closes the io.ReadCloser with error capture but exhausts the reader before. @@ -156,51 +151,5 @@ func ExhaustCloseWithErrCapture(err *error, r io.ReadCloser, format string, a .. CloseWithErrCapture(err, r, format, a...) // Prepend the io.Copy error. - merr := MultiError{} - merr.Add(copyErr) - merr.Add(*err) - - *err = merr.Err() -} - -// The MultiError type implements the error interface, and contains the -// Errors used to construct it. -type MultiError []error - -// Returns a concatenated string of the contained errors. -func (es MultiError) Error() string { - var buf bytes.Buffer - - if len(es) > 1 { - fmt.Fprintf(&buf, "%d errors: ", len(es)) - } - - for i, err := range es { - if i != 0 { - buf.WriteString("; ") - } - buf.WriteString(err.Error()) - } - - return buf.String() -} - -// Add adds the error to the error list if it is not nil. -func (es *MultiError) Add(err error) { - if err == nil { - return - } - if merr, ok := err.(MultiError); ok { - *es = append(*es, merr...) - } else { - *es = append(*es, err) - } -} - -// Err returns the error list as an error or nil if it is empty. -func (es MultiError) Err() error { - if len(es) == 0 { - return nil - } - return es + *err = merrors.New(copyErr, *err).Err() } diff --git a/scripts/cleanup-white-noise.sh b/scripts/cleanup-white-noise.sh deleted file mode 100755 index 9cd366b..0000000 --- a/scripts/cleanup-white-noise.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -SED_BIN=${SED_BIN:-sed} - -${SED_BIN} -i 's/[ \t]*$//' "$@"