diff --git a/go.mod b/go.mod index 2ff7f51..0516087 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aaronland/go-artisanal-integers-proxy v0.2.5 github.com/aaronland/go-pool v1.0.0 github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/saracen/walker v0.1.2 github.com/sfomuseum/go-edtf v0.2.3 github.com/smartystreets/scanners v1.0.4 github.com/tidwall/gjson v1.8.1 @@ -13,7 +14,6 @@ require ( github.com/whosonfirst/go-whosonfirst-export v0.4.0 github.com/whosonfirst/go-whosonfirst-geojson-v2 v0.16.3 github.com/whosonfirst/go-whosonfirst-id-proxy v0.0.1 - github.com/whosonfirst/go-whosonfirst-index v0.3.4 github.com/whosonfirst/go-whosonfirst-uri v1.1.0 github.com/whosonfirst/warning v0.1.1 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c @@ -22,7 +22,6 @@ require ( require ( github.com/aaronland/go-artisanal-integers v0.1.1 // indirect github.com/aaronland/go-brooklynintegers-api v1.2.2 // indirect - github.com/aaronland/go-json-query v0.0.2 // indirect github.com/aaronland/go-londonintegers-api v0.1.1 // indirect github.com/aaronland/go-missionintegers-api v0.1.1 // indirect github.com/aaronland/go-roster v0.0.2 // indirect @@ -38,7 +37,6 @@ require ( github.com/skelterjohn/geom v0.0.0-20180103142417-96f3e8a219c5 // indirect github.com/tidwall/match v1.0.3 // indirect github.com/tidwall/pretty v1.2.0 // indirect - github.com/whosonfirst/go-whosonfirst-crawl v0.2.1 // indirect github.com/whosonfirst/go-whosonfirst-flags v0.4.3 // indirect github.com/whosonfirst/go-whosonfirst-format v0.3.6 // indirect github.com/whosonfirst/go-whosonfirst-hash v0.1.0 // indirect @@ -47,7 +45,6 @@ require ( github.com/whosonfirst/go-whosonfirst-placetypes v0.3.0 // indirect github.com/whosonfirst/go-whosonfirst-sources v0.1.0 // indirect github.com/whosonfirst/go-whosonfirst-spr/v2 v2.0.0 // indirect - github.com/whosonfirst/walk v0.0.1 // indirect go.uber.org/ratelimit v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index 6342b05..b68acdf 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/aaronland/go-brooklynintegers-api v1.1.0/go.mod h1:agceH8JW8PcdHGCyKE github.com/aaronland/go-brooklynintegers-api v1.2.1/go.mod h1:vFyhY89Uc0LDZEaEam5EQ+2La2VlkTvSfExuZgZhmX4= github.com/aaronland/go-brooklynintegers-api v1.2.2 h1:rZmXfSBA1ynQZH/MsS2sCgo1NFHDuz7OQJw3U1RKhe8= github.com/aaronland/go-brooklynintegers-api v1.2.2/go.mod h1:mTWOMd95OM9EwjgTP/eyNoLQh1bfzDU4nIoZGhrOnyc= -github.com/aaronland/go-json-query v0.0.2 h1:cKw/DnxtGaPsClb78ONgmXPOJkqHhaBMU2RTBbPoXos= -github.com/aaronland/go-json-query v0.0.2/go.mod h1:dGc7y824R93ugQMTldL7PFD/SPVGPwKdqrUJFLE/ILU= github.com/aaronland/go-londonintegers-api v0.1.0/go.mod h1:E0VIcwks+So4FL1FYAQzrO5/OVOUsFPsqW85UrZArSE= github.com/aaronland/go-londonintegers-api v0.1.1 h1:/pRWI1FvcOuxsTkK4HY6xUiMPWQlQC7aozJTmf6f2Rk= github.com/aaronland/go-londonintegers-api v0.1.1/go.mod h1:N7Y0jrx+ouRAMrFcdH4nvJH6vrszj8/3yOOe0F8EV1E= @@ -71,6 +69,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/saracen/walker v0.1.2 h1:/o1TxP82n8thLvmL4GpJXduYaRmJ7qXp8u9dSlV0zmo= +github.com/saracen/walker v0.1.2/go.mod h1:0oKYMsKVhSJ+ful4p/XbjvXbMgLEkLITZaxozsl4CGE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sfomuseum/go-edtf v0.2.2/go.mod h1:1rP0EJZ/84j3HO80vGcnG2T9MFBDAFyTNtjrr8cv3T4= github.com/sfomuseum/go-edtf v0.2.3 h1:wpcpwl1RD9W/sXFDi4zpoIpQcIwIk8em9CGwa7YWv4g= @@ -108,8 +108,6 @@ github.com/tidwall/sjson v1.1.7 h1:sgVPwu/yygHJ2m1pJDLgGM/h+1F5odx5Q9ljG3imRm8= github.com/tidwall/sjson v1.1.7/go.mod h1:w/yG+ezBeTdUxiKs5NcPicO9diP38nk96QBAbIIGeFs= github.com/whosonfirst/algnhsa v0.1.0/go.mod h1:swLBXxaVTv3s6dJLhekdQCuCTshUew+xHjptRC21RG0= github.com/whosonfirst/go-whosonfirst-cli v0.1.0/go.mod h1:Edy+amD+fMq1QS1yxB3u8maA8I93q/LG7JRNh+fsdfc= -github.com/whosonfirst/go-whosonfirst-crawl v0.2.1 h1:nNG7r7/4MaII/NM8Df2oqgfgVNBDoIKlseleoX1vw1Q= -github.com/whosonfirst/go-whosonfirst-crawl v0.2.1/go.mod h1:MTD1TCgAkXlAtysPU98ylrz9Y5+ZCfRrsrBnRyiH/t8= github.com/whosonfirst/go-whosonfirst-export v0.4.0 h1:d+2V+wGjRH2ldCLCFb/JTl2jYGEBxSaWZlwDDeWfI8U= github.com/whosonfirst/go-whosonfirst-export v0.4.0/go.mod h1:XmkUCwpzlHNUWURdOeWcDrDQlWmdNe09lIvVIzrpYCI= github.com/whosonfirst/go-whosonfirst-flags v0.2.0/go.mod h1:ECd0AJJZIlybmjTGB9z+CPz9pSiMTwxur7fPKmDnoqI= @@ -127,8 +125,6 @@ github.com/whosonfirst/go-whosonfirst-id v0.0.2 h1:MYtBPYtOND/KSLYeaoURWyh2OkyVq github.com/whosonfirst/go-whosonfirst-id v0.0.2/go.mod h1:/Oq+Gbvlf33mixjc1aMqAAaOh1JB+wWlNRptbSqP0vI= github.com/whosonfirst/go-whosonfirst-id-proxy v0.0.1 h1:nTGnC6ZHWwt4w4MwXlJ5SpHDzOdoL+8vFkXyaKIm4X0= github.com/whosonfirst/go-whosonfirst-id-proxy v0.0.1/go.mod h1:53NH695H1SYDLAYbouOZ7gTTvHA9DuEarhikyuP3sQs= -github.com/whosonfirst/go-whosonfirst-index v0.3.4 h1:Ix6IxQOt3FEDcsgtD7uf/cfaKS9AkOgvFePkpWHWIFc= -github.com/whosonfirst/go-whosonfirst-index v0.3.4/go.mod h1:6PU507JqA3wK/WbCeA3jVGMqcBD/QZ6JGbzmDEXdlfU= github.com/whosonfirst/go-whosonfirst-log v0.1.0 h1:mWYI5hn16uyeLxBmPsLSvYV4rQKK/cxGVhM+bC2ZoGc= github.com/whosonfirst/go-whosonfirst-log v0.1.0/go.mod h1:pmgBbxZSnjGVy2nsUJBBMcFagxwIKLlmRsW7ClkXmac= github.com/whosonfirst/go-whosonfirst-placetypes v0.2.4/go.mod h1:yl0zZ5tfK80C0kl34pJcPB3mZC5XXR7ybQJ5OJyEcDU= @@ -143,8 +139,6 @@ github.com/whosonfirst/go-whosonfirst-uri v0.2.0/go.mod h1:8eaDVcc4v+HHHEDaRbApd github.com/whosonfirst/go-whosonfirst-uri v1.0.1/go.mod h1:8eaDVcc4v+HHHEDaRbApdmhPwM4/JQllw2PktvZcPVs= github.com/whosonfirst/go-whosonfirst-uri v1.1.0 h1:kNYOmKSm3u2asUOeq7yXL1Q8gFKkPIl8A0qinKrGV/8= github.com/whosonfirst/go-whosonfirst-uri v1.1.0/go.mod h1:8eaDVcc4v+HHHEDaRbApdmhPwM4/JQllw2PktvZcPVs= -github.com/whosonfirst/walk v0.0.1 h1:t0QrqGwOdPMSeovFZSXfiS0GIGHrRXK3Wb9z5Uhs2bg= -github.com/whosonfirst/walk v0.0.1/go.mod h1:1KtP/VeooSlFOI61p+THc/C16Ra8Z5MjpjI0tsd3c1M= github.com/whosonfirst/warning v0.1.1 h1:h29zL3VNL9VUHztkAAndzblhrDHyik9z47OuUR2Vovw= github.com/whosonfirst/warning v0.1.1/go.mod h1:/unEMzhB9YaMeEwTJpzLN3kM5LiSxdJhKEsf/OQhn6s= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -168,6 +162,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/github.com/aaronland/go-json-query/.gitignore b/vendor/github.com/aaronland/go-json-query/.gitignore deleted file mode 100644 index e4e5f6c..0000000 --- a/vendor/github.com/aaronland/go-json-query/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*~ \ No newline at end of file diff --git a/vendor/github.com/aaronland/go-json-query/README.md b/vendor/github.com/aaronland/go-json-query/README.md deleted file mode 100644 index 0d88a23..0000000 --- a/vendor/github.com/aaronland/go-json-query/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# go-json-query - diff --git a/vendor/github.com/aaronland/go-json-query/flags.go b/vendor/github.com/aaronland/go-json-query/flags.go deleted file mode 100644 index 773997b..0000000 --- a/vendor/github.com/aaronland/go-json-query/flags.go +++ /dev/null @@ -1,41 +0,0 @@ -package query - -import ( - "errors" - "regexp" - "strings" -) - -const SEP string = "=" - -type QueryFlags []*Query - -func (m *QueryFlags) String() string { - return "" -} - -func (m *QueryFlags) Set(value string) error { - - parts := strings.Split(value, SEP) - - if len(parts) != 2 { - return errors.New("Invalid query flag") - } - - path := parts[0] - str_match := parts[1] - - re, err := regexp.Compile(str_match) - - if err != nil { - return err - } - - q := &Query{ - Path: path, - Match: re, - } - - *m = append(*m, q) - return nil -} diff --git a/vendor/github.com/aaronland/go-json-query/query.go b/vendor/github.com/aaronland/go-json-query/query.go deleted file mode 100644 index 786a5b6..0000000 --- a/vendor/github.com/aaronland/go-json-query/query.go +++ /dev/null @@ -1,79 +0,0 @@ -package query - -import ( - "context" - "github.com/tidwall/gjson" - _ "log" - "regexp" -) - -const QUERYSET_MODE_ANY string = "ANY" -const QUERYSET_MODE_ALL string = "ALL" - -type QuerySet struct { - Queries []*Query - Mode string -} - -type Query struct { - Path string - Match *regexp.Regexp -} - -func Matches(ctx context.Context, qs *QuerySet, body []byte) (bool, error) { - - select { - case <-ctx.Done(): - return false, nil - default: - // pass - } - - queries := qs.Queries - mode := qs.Mode - - tests := len(queries) - matches := 0 - - for _, q := range queries { - - rsp := gjson.GetBytes(body, q.Path) - - if !rsp.Exists() { - - if mode == QUERYSET_MODE_ALL { - break - } - } - - for _, r := range rsp.Array() { - - if q.Match.MatchString(r.String()) { - - matches += 1 - - if mode == QUERYSET_MODE_ANY { - break - } - } - } - - if mode == QUERYSET_MODE_ANY && matches > 0 { - break - } - - } - - if mode == QUERYSET_MODE_ALL { - - if matches < tests { - return false, nil - } - } - - if matches == 0 { - return false, nil - } - - return true, nil -} diff --git a/vendor/github.com/saracen/walker/LICENSE b/vendor/github.com/saracen/walker/LICENSE new file mode 100644 index 0000000..25944ed --- /dev/null +++ b/vendor/github.com/saracen/walker/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Arran Walker + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/saracen/walker/README.md b/vendor/github.com/saracen/walker/README.md new file mode 100644 index 0000000..7bed5d7 --- /dev/null +++ b/vendor/github.com/saracen/walker/README.md @@ -0,0 +1,104 @@ +# walker + +[![](https://godoc.org/github.com/saracen/walker?status.svg)](http://godoc.org/github.com/saracen/walker) + +`walker` is a faster, parallel version, of `filepath.Walk`. + +```go +// walk function called for every path found +walkFn := func(pathname string, fi os.FileInfo) error { + fmt.Printf("%s: %d bytes\n", pathname, fi.Size()) + return nil +} + +// error function called for every error encountered +errorCallbackOption := walker.WithErrorCallback(func(pathname string, err error) error { + // ignore permissione errors + if os.IsPermission(err) { + return nil + } + // halt traversal on any other error + return err +}) + +walker.Walk("/tmp", walkFn, errorCallbackOption) +``` + +## Benchmarks + +- Standard library (`filepath.Walk`) is `FilepathWalk`. +- This library is `WalkerWalk` +- `FastwalkWalk` is [fastwalk](https://github.com/golang/tools/tree/master/internal/fastwalk). +- `GodirwalkWalk` is [godirwalk](https://github.com/karrick/godirwalk). + +This library and `filepath.Walk` both perform `os.Lstat` calls and provide a full `os.FileInfo` structure to the callback. `BenchmarkFastwalkWalkLstat` and `BenchmarkGodirwalkWalkLstat` include this stat call for better comparison with `BenchmarkFilepathWalk` and `BenchmarkWalkerWalk`. + +This library and `fastwalk` both require the callback to be safe for concurrent use. `BenchmarkFilepathWalkAppend`, `BenchmarkWalkerWalkAppend`, `BenchmarkFastwalkWalkAppend` and `BenchmarkGodirwalkWalkAppend` append the paths found to a string slice. The callback, for the libraries that require it, use a mutex, for better comparison with the libraries that require no locking. + +This library will not always be the best/fastest option. In general, if you're on Windows, or performing `lstat` calls, it does a pretty decent job. If you're not, I've found `fastwalk` to perform better on machines with fewer cores. + +These benchmarks were performed with a warm cache. + +``` +goos: linux +goarch: amd64 +pkg: github.com/saracen/walker +BenchmarkFilepathWalk-16 1 1437919955 ns/op 340100304 B/op 775525 allocs/op +BenchmarkFilepathWalkAppend-16 1 1226169600 ns/op 351722832 B/op 775556 allocs/op +BenchmarkWalkerWalk-16 8 133364860 ns/op 92611308 B/op 734674 allocs/op +BenchmarkWalkerWalkAppend-16 7 166917499 ns/op 104231474 B/op 734693 allocs/op +BenchmarkFastwalkWalk-16 6 241763690 ns/op 25257176 B/op 309423 allocs/op +BenchmarkFastwalkWalkAppend-16 4 285673715 ns/op 36898800 B/op 309456 allocs/op +BenchmarkFastwalkWalkLstat-16 6 176641625 ns/op 73769765 B/op 592980 allocs/op +BenchmarkGodirwalkWalk-16 2 714625929 ns/op 145340576 B/op 723225 allocs/op +BenchmarkGodirwalkWalkAppend-16 2 597653802 ns/op 156963288 B/op 723256 allocs/op +BenchmarkGodirwalkWalkLstat-16 1 1186956102 ns/op 193724464 B/op 1006727 allocs/op +``` + +``` +goos: windows +goarch: amd64 +pkg: github.com/saracen/walker +BenchmarkFilepathWalk-16 1 1268606000 ns/op 101248040 B/op 650718 allocs/op +BenchmarkFilepathWalkAppend-16 1 1276617400 ns/op 107079288 B/op 650744 allocs/op +BenchmarkWalkerWalk-16 12 98901983 ns/op 52393125 B/op 382836 allocs/op +BenchmarkWalkerWalkAppend-16 12 99733117 ns/op 58220869 B/op 382853 allocs/op +BenchmarkFastwalkWalk-16 10 109107980 ns/op 53032702 B/op 401320 allocs/op +BenchmarkFastwalkWalkAppend-16 10 107512330 ns/op 58853827 B/op 401336 allocs/op +BenchmarkFastwalkWalkLstat-16 3 379318333 ns/op 100606232 B/op 653931 allocs/op +BenchmarkGodirwalkWalk-16 3 466418533 ns/op 42955197 B/op 579974 allocs/op +BenchmarkGodirwalkWalkAppend-16 3 476391833 ns/op 48786530 B/op 580002 allocs/op +BenchmarkGodirwalkWalkLstat-16 1 1250652800 ns/op 90536184 B/op 832562 allocs/op +``` + +Performing benchmarks without having the OS cache the directory information isn't straight forward, but to get a sense of the performance, we can flush the cache and roughly time how long it took to walk a directory: + +#### filepath.Walk +``` +$ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFilepathWalkDir -benchdir $GOPATH +ok github.com/saracen/walker 3.846s +``` + +#### walker +``` +$ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestWalkerWalkDir -benchdir $GOPATH +ok github.com/saracen/walker 0.353s +``` + +#### fastwalk +``` +$ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFastwalkWalkDir -benchdir $GOPATH +ok github.com/saracen/walker 0.306s +``` + +#### fastwalk (lstat) +``` +$ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestFastwalkWalkLstatDir -benchdir $GOPATH +ok github.com/saracen/walker 0.339s +``` + +#### godirwalk +``` +$ sudo su -c 'sync; echo 3 > /proc/sys/vm/drop_caches'; go test -run TestGodirwalkWalkDir -benchdir $GOPATH +ok github.com/saracen/walker 3.208s +``` diff --git a/vendor/github.com/saracen/walker/walker.go b/vendor/github.com/saracen/walker/walker.go new file mode 100644 index 0000000..6431d33 --- /dev/null +++ b/vendor/github.com/saracen/walker/walker.go @@ -0,0 +1,125 @@ +package walker + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + + "golang.org/x/sync/errgroup" +) + +// Walk wraps WalkWithContext using the background context. +func Walk(root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...Option) error { + return WalkWithContext(context.Background(), root, walkFn, opts...) +} + +// WalkWithContext walks the file tree rooted at root, calling walkFn for each +// file or directory in the tree, including root. +// +// If fastWalk returns filepath.SkipDir, the directory is skipped. +// +// Multiple goroutines stat the filesystem concurrently. The provided +// walkFn must be safe for concurrent use. +func WalkWithContext(ctx context.Context, root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...Option) error { + wg, ctx := errgroup.WithContext(ctx) + + fi, err := os.Lstat(root) + if err != nil { + return err + } + if err = walkFn(root, fi); err == filepath.SkipDir { + return nil + } + if err != nil || !fi.IsDir() { + return err + } + + w := walker{ + counter: 1, + limit: runtime.NumCPU(), + ctx: ctx, + wg: wg, + fn: walkFn, + } + if w.limit < 4 { + w.limit = 4 + } + + for _, o := range opts { + err := o(&w.options) + if err != nil { + return err + } + } + + w.wg.Go(func() error { + return w.gowalk(root) + }) + + return w.wg.Wait() +} + +type walker struct { + counter uint32 + limit int + ctx context.Context + wg *errgroup.Group + fn func(pathname string, fi os.FileInfo) error + options walkerOptions +} + +func (w *walker) walk(dirname string, fi os.FileInfo) error { + pathname := dirname + string(filepath.Separator) + fi.Name() + + err := w.fn(pathname, fi) + if err == filepath.SkipDir { + return nil + } + if err != nil { + return err + } + + // don't follow symbolic links + if fi.Mode()&os.ModeSymlink != 0 { + return nil + } + + if !fi.IsDir() { + return nil + } + + if err = w.ctx.Err(); err != nil { + return err + } + + current := atomic.LoadUint32(&w.counter) + + // if we haven't reached our goroutine limit, spawn a new one + if current < uint32(w.limit) { + if atomic.CompareAndSwapUint32(&w.counter, current, current+1) { + w.wg.Go(func() error { + return w.gowalk(pathname) + }) + return nil + } + } + + // if we've reached our limit, continue with this goroutine + err = w.readdir(pathname) + if err != nil && w.options.errorCallback != nil { + err = w.options.errorCallback(pathname, err) + } + return err +} + +func (w *walker) gowalk(pathname string) error { + err := w.readdir(pathname) + if err != nil && w.options.errorCallback != nil { + err = w.options.errorCallback(pathname, err) + } + + atomic.AddUint32(&w.counter, ^uint32(0)) + return err +} diff --git a/vendor/github.com/saracen/walker/walker_option.go b/vendor/github.com/saracen/walker/walker_option.go new file mode 100644 index 0000000..b88ef9b --- /dev/null +++ b/vendor/github.com/saracen/walker/walker_option.go @@ -0,0 +1,18 @@ +package walker + +// WalkerOption is an option to configure Walk() behaviour. +type Option func(*walkerOptions) error + +type walkerOptions struct { + errorCallback func(pathname string, err error) error +} + +// WithErrorCallback sets a callback to be used for error handling. Any error +// returned will halt the Walk function and return the error. If the callback +// returns nil Walk will continue. +func WithErrorCallback(callback func(pathname string, err error) error) Option { + return func(o *walkerOptions) error { + o.errorCallback = callback + return nil + } +} diff --git a/vendor/github.com/saracen/walker/walker_portable.go b/vendor/github.com/saracen/walker/walker_portable.go new file mode 100644 index 0000000..3dc22ac --- /dev/null +++ b/vendor/github.com/saracen/walker/walker_portable.go @@ -0,0 +1,25 @@ +// +build appengine !linux,!darwin,!freebsd,!openbsd,!netbsd + +package walker + +import "os" + +func (w *walker) readdir(dirname string) error { + f, err := os.Open(dirname) + if err != nil { + return err + } + + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return err + } + + for _, fi := range list { + if err = w.walk(dirname, fi); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/saracen/walker/walker_unix.go b/vendor/github.com/saracen/walker/walker_unix.go new file mode 100644 index 0000000..7048aa3 --- /dev/null +++ b/vendor/github.com/saracen/walker/walker_unix.go @@ -0,0 +1,76 @@ +// +build linux darwin freebsd openbsd netbsd +// +build !appengine + +package walker + +import ( + "os" + "syscall" +) + +func (w *walker) readdir(dirname string) error { + fd, err := open(dirname, 0, 0) + if err != nil { + return &os.PathError{Op: "open", Path: dirname, Err: err} + } + defer syscall.Close(fd) + + buf := make([]byte, 8<<10) + names := make([]string, 0, 100) + + nbuf := 0 + bufp := 0 + for { + if bufp >= nbuf { + bufp = 0 + nbuf, err = readDirent(fd, buf) + if err != nil { + return err + } + if nbuf <= 0 { + return nil + } + } + + consumed, count, names := syscall.ParseDirent(buf[bufp:nbuf], 100, names[0:]) + bufp += consumed + + for _, name := range names[:count] { + fi, err := os.Lstat(dirname + "/" + name) + if os.IsNotExist(err) { + continue + } + if err != nil { + return err + } + if err = w.walk(dirname, fi); err != nil { + return err + } + } + } + // never reach +} + +// According to https://golang.org/doc/go1.14#runtime +// A consequence of the implementation of preemption is that on Unix systems, including Linux and macOS +// systems, programs built with Go 1.14 will receive more signals than programs built with earlier releases. +// +// This causes syscall.Open and syscall.ReadDirent sometimes fail with EINTR errors. +// We need to retry in this case. +func open(path string, mode int, perm uint32) (fd int, err error) { + for { + fd, err := syscall.Open(path, mode, perm) + if err != syscall.EINTR { + return fd, err + } + } +} + +func readDirent(fd int, buf []byte) (n int, err error) { + for { + nbuf, err := syscall.ReadDirent(fd, buf) + if err != syscall.EINTR { + return nbuf, err + } + } +} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/.gitignore b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/.gitignore deleted file mode 100644 index 897a158..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*~ -pkg -src -bin/wof-* -!vendor/src \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/LICENSE b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/LICENSE deleted file mode 100644 index 36cd097..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2015, Mapzen -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/Makefile b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/Makefile deleted file mode 100644 index 85cc50b..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -fmt: - go fmt ./... - -tools: - go build -o bin/wof-count cmd/wof-count/main.go diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/README.md b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/README.md deleted file mode 100644 index 297a89d..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# go-mapzen-whosonfirst-crawl - -Go tools and libraries for crawling a directory of Who's On First data - -## Install - -You will need to have both `Go` (specifically version [1.12](https://golang.org/dl/) or higher) and the `make` programs installed on your computer. Assuming you do just type: - -``` -make tools -``` - -All of this package's dependencies are bundled with the code in the `vendor` directory. - -## Example - -### crawl.Crawl - -``` -package main - -import ( - "flag" - "fmt" - "github.com/whosonfirst/go-whosonfirst-crawl" - "log" - "os" - "time" - "sync/atomic" -) - -func main() { - - root := flag.String("root", "", "The root directory you want to crawl") - - flag.Parse() - - var files int64 - var dirs int64 - - callback := func(path string, info os.FileInfo) error { - - if info.IsDir() { - atomic.AddInt64(&dirs, 1) - return nil - } - - atomic.AddInt64(&files, 1) - return nil - } - - t0 := time.Now() - - defer func(){ - t1 := float64(time.Since(t0)) / 1e9 - fmt.Printf("walked %d files (and %d dirs) in %s in %.3f seconds\n", files, dirs, *root, t1) - }() - - c := crawl.NewCrawler(*root) - err := c.Crawl(callback) - - if err != nil { - log.Fatal(err) - } -} -``` - -### crawl.CrawlWithContext - -_Please write me_ - -### crawl.CrawlWithChannels - -_Please write me_ - -## Tools - -### wof-count - -``` -./bin/wof-count /usr/local/data/sfomuseum-data-flights-2019-* -go build -o bin/wof-count cmd/wof-count/main.go -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-01 -walked 98116 files (and 0 dirs) in 4.203 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-02 -walked 87989 files (and 0 dirs) in 4.969 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-03 -walked 102354 files (and 0 dirs) in 6.178 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-04 -walked 107775 files (and 0 dirs) in 6.377 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-05 -walked 143200 files (and 0 dirs) in 7.565 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-06 -walked 124490 files (and 0 dirs) in 6.326 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-07 -walked 63 files (and 0 dirs) in 0.007 seconds -count files and directories in /usr/local/data/sfomuseum-data-flights-2019-08 -walked 43 files (and 0 dirs) in 0.005 seconds -``` - -## See also - -* https://github.com/whosonfirst/walk \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/crawl.go b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/crawl.go deleted file mode 100644 index dd9dfd9..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/crawl.go +++ /dev/null @@ -1,151 +0,0 @@ -package crawl - -import ( - "context" - walk "github.com/whosonfirst/walk" - _ "log" - "os" -) - -type ProcessingRequest struct { - Path string - Ready chan bool -} - -type CrawlFunc func(path string, info os.FileInfo) error - -type RequestHandlerFunc func(req *ProcessingRequest) bool - -type Crawler struct { - Root string - CrawlDirectories bool -} - -func NewCrawler(path string) *Crawler { - return &Crawler{ - Root: path, - CrawlDirectories: false, - } -} - -func (c Crawler) Crawl(cb CrawlFunc) error { - - ctx := context.Background() - return c.CrawlWithContext(ctx, cb) -} - -func (c Crawler) CrawlWithContext(ctx context.Context, cb CrawlFunc) error { - - req_handler := func(_ *ProcessingRequest) bool { - return true - } - - return c.CrawlWithContextAndRequestHandler(ctx, cb, req_handler) -} - -func (c Crawler) CrawlWithContextAndRequestHandler(ctx context.Context, cb CrawlFunc, req_handler RequestHandlerFunc) error { - - // this bit is important - see abouts about ctx.Done() and DoneError() - // below in CrawlWithChannels (20190822/thisisaaronland) - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - processing_ch := make(chan *ProcessingRequest) - error_ch := make(chan error) - done_ch := make(chan bool) - - go c.CrawlWithChannels(ctx, cb, processing_ch, error_ch, done_ch) - - for { - select { - case <-done_ch: - return nil - case req := <-processing_ch: - req.Ready <- req_handler(req) // see notes about "processing requests" in CrawlWithChannels - case err := <-error_ch: - return err - } - } -} - -func (c Crawler) CrawlWithChannels(ctx context.Context, cb CrawlFunc, processing_ch chan *ProcessingRequest, error_ch chan error, done_ch chan bool) { - - defer func() { - done_ch <- true - }() - - // note the bit with the DoneError() - if the `context.Context` object has signaled - // that we're done we want to stop processing files but the only way to do that is - // to send the `walk.Walk` object an error. In this case it's a special "done" error - // that is not bubbled back up the stack to the caller (20190822/thisisaaronland) - - walker := func(path string, info os.FileInfo, err error) error { - - select { - case <-ctx.Done(): - return NewDoneError() - default: - // pass - } - - if err != nil { - error_ch <- NewWalkError(path, err) - return nil - } - - if info.IsDir() && !c.CrawlDirectories { - return nil - } - - // processing request allows people using this package - // implement their own custom throttling - here we send - // the "processing" channel a request that contains a local - // "ready" channel and then we wait for a response. If - // the response is true then we carry on but if it's false - // we exit out of the function with a nil return value - - ready_ch := make(chan bool) - ready := false - - req := &ProcessingRequest{ - Path: path, - Ready: ready_ch, - } - - processing_ch <- req - - for { - select { - case ready_value := <-ready_ch: - - if ready_value == false { - return nil - } - - ready = true - } - - if ready { - break - } - } - - // okay, carry on - - err = cb(path, info) - - if err != nil { - error_ch <- NewCallbackError(path, err) - return nil - } - - return nil - } - - err := walk.Walk(c.Root, walker) - - if err != nil && !IsDoneError(err) { - error_ch <- NewCrawlError(c.Root, err) - } -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/errors.go b/vendor/github.com/whosonfirst/go-whosonfirst-crawl/errors.go deleted file mode 100644 index b5e1150..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-crawl/errors.go +++ /dev/null @@ -1,124 +0,0 @@ -package crawl - -import ( - "fmt" -) - -type DoneError struct{} - -func (e *DoneError) Error() string { - return "context.Context signaled Done()" -} - -type CrawlError struct { - Path string - Details error -} - -func (e *CrawlError) Error() string { - return e.String() -} - -func (e *CrawlError) String() string { - return fmt.Sprintf("Failed crawl for %s: %v", e.Path, e.Details) -} - -type WalkError struct { - Path string - Details error -} - -func (e *WalkError) Error() string { - return e.String() -} - -func (e *WalkError) String() string { - return fmt.Sprintf("Failed walk for %s: %v", e.Path, e.Details) -} - -type CallbackError struct { - Path string - Details error -} - -func (e *CallbackError) Error() string { - return e.String() -} - -func (e *CallbackError) String() string { - return fmt.Sprintf("Failed crawl callback for %s: %v", e.Path, e.Details) -} - -func NewDoneError() *DoneError { - return &DoneError{} -} - -func NewCrawlError(path string, details error) *CrawlError { - - err := CrawlError{ - Path: path, - Details: details, - } - - return &err -} - -func NewWalkError(path string, details error) *WalkError { - - err := WalkError{ - Path: path, - Details: details, - } - - return &err -} - -func NewCallbackError(path string, details error) *CallbackError { - - err := CallbackError{ - Path: path, - Details: details, - } - - return &err -} - -func IsDoneError(err error) bool { - - switch err.(type) { - case *DoneError: - return true - default: - return false - } -} - -func IsCrawlError(err error) bool { - - switch err.(type) { - case *CrawlError: - return true - default: - return false - } -} - -func IsWalkError(err error) bool { - - switch err.(type) { - case *WalkError: - return true - default: - return false - } -} - -func IsCallbackError(err error) bool { - - switch err.(type) { - case *CallbackError: - return true - default: - return false - } -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/.gitignore b/vendor/github.com/whosonfirst/go-whosonfirst-index/.gitignore deleted file mode 100644 index 3e3520e..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*~ -pkg -src -!vendor/src -bin -!bin/.gitignore -*.log -*.json -.travis.yml \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/LICENSE b/vendor/github.com/whosonfirst/go-whosonfirst-index/LICENSE deleted file mode 100644 index ea18d16..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2017, Mapzen -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/Makefile b/vendor/github.com/whosonfirst/go-whosonfirst-index/Makefile deleted file mode 100644 index b08b11b..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -tools: - go build -mod vendor -o bin/wof-index-count cmd/wof-index-count/main.go diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/README.md b/vendor/github.com/whosonfirst/go-whosonfirst-index/README.md deleted file mode 100644 index 1286709..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# go-whosonfirst-index - -Go package for indexing Who's On First documents - -## Install - -You will need to have both `Go` (specifically a version [1.12](https://golang.org/dl/) or higher) and the `make` programs installed on your computer. Assuming you do just type: - -``` -make tools -``` - -All of this package's dependencies are bundled with the code in the `vendor` directory. - -## Example - -``` -package main - -import ( - "context" - "flag" - "github.com/whosonfirst/go-whosonfirst-index" - _ "github.com/whosonfirst/go-whosonfirst-index/fs" - "io" - "log" -) - -func main() { - - var dsn = flag.String("dsn", "repo://", "A valid go-whosonfirst-index DSN") - - flag.Parse() - - cb := func(ctx context.Context, fh io.Reader, args ...interface{}) error { - - path, _ := index.PathForContext(ctx) - - log.Println("PATH", path) - return nil - } - - i, _ := index.NewIndexer(*dsn, cb) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - paths := flag.Args() - - i.Index(ctx, paths...) -} -``` - -_Error handling removed for the sake of brevity._ diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/driver.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/driver.go deleted file mode 100644 index d9b7693..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/driver.go +++ /dev/null @@ -1,10 +0,0 @@ -package index - -import ( - "context" -) - -type Driver interface { - Open(string) error - IndexURI(context.Context, IndexerFunc, string) error -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/directory.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/directory.go deleted file mode 100644 index 9ad9b7b..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/directory.go +++ /dev/null @@ -1,63 +0,0 @@ -package fs - -import ( - "context" - "github.com/whosonfirst/go-whosonfirst-crawl" - "github.com/whosonfirst/go-whosonfirst-index" - "os" - "path/filepath" -) - -func init() { - dr := NewDirectoryDriver() - index.Register("directory", dr) -} - -func NewDirectoryDriver() index.Driver { - return &DirectoryDriver{} -} - -type DirectoryDriver struct { - index.Driver -} - -func (d *DirectoryDriver) Open(uri string) error { - return nil -} - -func (d *DirectoryDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - abs_path, err := filepath.Abs(uri) - - if err != nil { - return err - } - - crawl_cb := func(path string, info os.FileInfo) error { - - select { - case <-ctx.Done(): - return nil - default: - // pass - } - - if info.IsDir() { - return nil - } - - fh, err := readerFromPath(path) - - if err != nil { - return err - } - - defer fh.Close() - - ctx = index.AssignPathContext(ctx, path) - return index_cb(ctx, fh) - } - - c := crawl.NewCrawler(abs_path) - return c.Crawl(crawl_cb) -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/featurecollection.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/featurecollection.go deleted file mode 100644 index b53ff34..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/featurecollection.go +++ /dev/null @@ -1,86 +0,0 @@ -package fs - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/whosonfirst/go-whosonfirst-index" - "io/ioutil" -) - -func init() { - dr := NewFeatureCollectionDriver() - index.Register("featurecollection", dr) -} - -func NewFeatureCollectionDriver() index.Driver { - return &FeatureCollectionDriver{} -} - -type FeatureCollectionDriver struct { - index.Driver -} - -func (d *FeatureCollectionDriver) Open(uri string) error { - return nil -} - -func (d *FeatureCollectionDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - fh, err := readerFromPath(uri) - - if err != nil { - return err - } - - defer fh.Close() - - body, err := ioutil.ReadAll(fh) - - if err != nil { - return err - } - - type FC struct { - Type string - Features []interface{} - } - - var collection FC - - err = json.Unmarshal(body, &collection) - - if err != nil { - return err - } - - for i, f := range collection.Features { - - select { - case <-ctx.Done(): - break - default: - // pass - } - - feature, err := json.Marshal(f) - - if err != nil { - return err - } - - fh := bytes.NewBuffer(feature) - - path := fmt.Sprintf("%s#%d", uri, i) - ctx = index.AssignPathContext(ctx, path) - - err = index_cb(ctx, fh) - - if err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/file.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/file.go deleted file mode 100644 index c3cb296..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/file.go +++ /dev/null @@ -1,38 +0,0 @@ -package fs - -import ( - "context" - "github.com/whosonfirst/go-whosonfirst-index" -) - -func init() { - dr := NewFileDriver() - index.Register("file", dr) -} - -func NewFileDriver() index.Driver { - return &FileDriver{} -} - -type FileDriver struct { - index.Driver -} - -func (d *FileDriver) Open(uri string) error { - return nil -} - -func (d *FileDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - fh, err := readerFromPath(uri) - - if err != nil { - return err - } - - defer fh.Close() - - ctx = index.AssignPathContext(ctx, uri) - return index_cb(ctx, fh) - -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/filelist.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/filelist.go deleted file mode 100644 index 8b8d1d0..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/filelist.go +++ /dev/null @@ -1,71 +0,0 @@ -package fs - -import ( - "bufio" - "context" - "github.com/whosonfirst/go-whosonfirst-index" -) - -func init() { - dr := NewFileListDriver() - index.Register("filelist", dr) -} - -func NewFileListDriver() index.Driver { - return &FileListDriver{} -} - -type FileListDriver struct { - index.Driver -} - -func (d *FileListDriver) Open(uri string) error { - return nil -} - -func (d *FileListDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - fh, err := readerFromPath(uri) - - if err != nil { - return err - } - - defer fh.Close() - - scanner := bufio.NewScanner(fh) - - for scanner.Scan() { - - select { - case <-ctx.Done(): - break - default: - // pass - } - - path := scanner.Text() - - fh, err := readerFromPath(path) - - if err != nil { - return err - } - - ctx = index.AssignPathContext(ctx, path) - - err = index_cb(ctx, fh) - - if err != nil { - return err - } - } - - err = scanner.Err() - - if err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/fs.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/fs.go deleted file mode 100644 index 895d6e9..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/fs.go +++ /dev/null @@ -1,23 +0,0 @@ -package fs - -import ( - "github.com/whosonfirst/go-whosonfirst-index" - "io" - "os" - _ "path/filepath" -) - -func readerFromPath(abs_path string) (io.ReadCloser, error) { - - if abs_path == index.STDIN { - return os.Stdin, nil - } - - fh, err := os.Open(abs_path) - - if err != nil { - return nil, err - } - - return fh, nil -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/geojsonls.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/geojsonls.go deleted file mode 100644 index 3c0e6a4..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/geojsonls.go +++ /dev/null @@ -1,90 +0,0 @@ -package fs - -import ( - "bufio" - "bytes" - "context" - "fmt" - "github.com/whosonfirst/go-whosonfirst-index" - "io" -) - -func init() { - dr := NewGeoJSONLDriver() - index.Register("geojsonl", dr) -} - -func NewGeoJSONLDriver() index.Driver { - return &GeojsonLDriver{} -} - -type GeojsonLDriver struct { - index.Driver -} - -func (d *GeojsonLDriver) Open(uri string) error { - return nil -} - -func (d *GeojsonLDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - fh, err := readerFromPath(uri) - - if err != nil { - return err - } - - defer fh.Close() - - // see this - we're using ReadLine because it's entirely possible - // that the raw GeoJSON (LS) will be too long for bufio.Scanner - // see also - https://golang.org/pkg/bufio/#Reader.ReadLine - // (20170822/thisisaaronland) - - reader := bufio.NewReader(fh) - raw := bytes.NewBuffer([]byte("")) - - i := 0 - - for { - - select { - case <-ctx.Done(): - break - default: - // pass - } - - path := fmt.Sprintf("%s#%d", uri, i) - i += 1 - - fragment, is_prefix, err := reader.ReadLine() - - if err == io.EOF { - break - } - - if err != nil { - return err - } - - raw.Write(fragment) - - if is_prefix { - continue - } - - fh := bytes.NewReader(raw.Bytes()) - - ctx = index.AssignPathContext(ctx, path) - err = index_cb(ctx, fh) - - if err != nil { - return err - } - - raw.Reset() - } - - return nil -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/repo.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/repo.go deleted file mode 100644 index 3e70d74..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/fs/repo.go +++ /dev/null @@ -1,45 +0,0 @@ -package fs - -import ( - "context" - "github.com/whosonfirst/go-whosonfirst-index" - "path/filepath" -) - -func init() { - dr := NewRepoDriver() - index.Register("repo", dr) -} - -func NewRepoDriver() index.Driver { - - dd := NewDirectoryDriver() - - dr := &RepoDriver{ - driver: dd, - } - - return dr -} - -type RepoDriver struct { - index.Driver - driver index.Driver -} - -func (d *RepoDriver) Open(uri string) error { - return d.driver.Open(uri) -} - -func (d *RepoDriver) IndexURI(ctx context.Context, index_cb index.IndexerFunc, uri string) error { - - abs_path, err := filepath.Abs(uri) - - if err != nil { - return err - } - - data_path := filepath.Join(abs_path, "data") - - return d.driver.IndexURI(ctx, index_cb, data_path) -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/index.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/index.go deleted file mode 100644 index be6c501..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/index.go +++ /dev/null @@ -1,224 +0,0 @@ -package index - -import ( - "context" - "errors" - "fmt" - "github.com/whosonfirst/go-whosonfirst-log" - "io" - "net/url" - "sort" - "sync" - "sync/atomic" - "time" -) - -const ( - STDIN = "STDIN" -) - -var ( - driversMu sync.RWMutex - drivers = make(map[string]Driver) -) - -type IndexerFunc func(ctx context.Context, fh io.Reader, args ...interface{}) error - -type IndexerContextKey string - -type Indexer struct { - Driver Driver - Func IndexerFunc - Logger *log.WOFLogger - Indexed int64 - count int64 -} - -func Register(name string, driver Driver) { - - driversMu.Lock() - defer driversMu.Unlock() - - if driver == nil { - panic("sql: Register driver is nil") - - } - - if _, dup := drivers[name]; dup { - panic("index: Register called twice for driver " + name) - } - - drivers[name] = driver -} - -func unregisterAllDrivers() { - driversMu.Lock() - defer driversMu.Unlock() - drivers = make(map[string]Driver) -} - -func Drivers() []string { - - driversMu.RLock() - defer driversMu.RUnlock() - - var list []string - - for name := range drivers { - list = append(list, name) - } - - sort.Strings(list) - return list -} - -func Modes() []string { - - return Drivers() -} - -func ContextForPath(path string) (context.Context, error) { - - ctx := AssignPathContext(context.Background(), path) - return ctx, nil -} - -func AssignPathContext(ctx context.Context, path string) context.Context { - - key := IndexerContextKey("path") - return context.WithValue(ctx, key, path) -} - -func PathForContext(ctx context.Context) (string, error) { - - k := IndexerContextKey("path") - path := ctx.Value(k) - - if path == nil { - return "", errors.New("path is not set") - } - - return path.(string), nil -} - -func NewIndexer(dsn string, f IndexerFunc) (*Indexer, error) { - - driversMu.Lock() - defer driversMu.Unlock() - - u, err := url.Parse(dsn) - - if err != nil { - return nil, err - } - - name := u.Scheme - - // this is here for backwards compatibility - - if name == "" { - - dsn = fmt.Sprintf("%s://", dsn) - - u, err := url.Parse(dsn) - - if err != nil { - return nil, err - } - - name = u.Scheme - } - - driver, ok := drivers[name] - - if !ok { - return nil, errors.New("Unknown driver") - } - - err = driver.Open(dsn) - - if err != nil { - return nil, err - } - - logger := log.SimpleWOFLogger("index") - - i := Indexer{ - Driver: driver, - Func: f, - Logger: logger, - Indexed: 0, - count: 0, - } - - return &i, nil -} - -func (i *Indexer) Index(ctx context.Context, paths ...string) error { - - t1 := time.Now() - - defer func() { - t2 := time.Since(t1) - i.Logger.Status("time to index paths (%d) %v", len(paths), t2) - }() - - i.increment() - defer i.decrement() - - counter_func := func(ctx context.Context, fh io.Reader, args ...interface{}) error { - defer atomic.AddInt64(&i.Indexed, 1) - return i.Func(ctx, fh, args...) - } - - for _, path := range paths { - - select { - case <-ctx.Done(): - break - default: - // pass - } - - err := i.Driver.IndexURI(ctx, counter_func, path) - - if err != nil { - return err - } - } - - return nil -} - -func (i *Indexer) IndexPaths(paths []string, args ...interface{}) error { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - return i.Index(ctx, paths...) -} - -func (i *Indexer) IndexPath(path string, args ...interface{}) error { - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - return i.Index(ctx, path) -} - -func (i *Indexer) IsIndexing() bool { - - if atomic.LoadInt64(&i.count) > 0 { - return true - } - - return false -} - -func (i *Indexer) increment() { - atomic.AddInt64(&i.count, 1) -} - -func (i *Indexer) decrement() { - atomic.AddInt64(&i.count, -1) -} diff --git a/vendor/github.com/whosonfirst/go-whosonfirst-index/query.go b/vendor/github.com/whosonfirst/go-whosonfirst-index/query.go deleted file mode 100644 index 6eab66a..0000000 --- a/vendor/github.com/whosonfirst/go-whosonfirst-index/query.go +++ /dev/null @@ -1,55 +0,0 @@ -package index - -import ( - "bytes" - "context" - "github.com/aaronland/go-json-query" - "io" - "io/ioutil" -) - -func NewCallbackWithQuerySet(ctx context.Context, indexer_cb IndexerFunc, qs *query.QuerySet) (IndexerFunc, error) { - - if qs == nil { - return indexer_cb, nil - } - - if len(qs.Queries) == 0 { - return indexer_cb, nil - } - - query_cb := func(ctx context.Context, fh io.Reader, args ...interface{}) error { - - body, err := ioutil.ReadAll(fh) - - if err != nil { - return err - } - - matches, err := query.Matches(ctx, qs, body) - - if err != nil { - return err - } - - if !matches { - return nil - } - - br := bytes.NewReader(body) - return indexer_cb(ctx, br, args...) - } - - return query_cb, nil -} - -func NewIndexerWithQuerySet(ctx context.Context, indexer_uri string, indexer_cb IndexerFunc, qs *query.QuerySet) (*Indexer, error) { - - query_cb, err := NewCallbackWithQuerySet(ctx, indexer_cb, qs) - - if err != nil { - return nil, err - } - - return NewIndexer(indexer_uri, query_cb) -} diff --git a/vendor/github.com/whosonfirst/walk/.gitignore b/vendor/github.com/whosonfirst/walk/.gitignore deleted file mode 100644 index 0ba233f..0000000 --- a/vendor/github.com/whosonfirst/walk/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*~ -pkg -src -.DS_Store \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/walk/LICENSE b/vendor/github.com/whosonfirst/walk/LICENSE deleted file mode 100644 index 81500c6..0000000 --- a/vendor/github.com/whosonfirst/walk/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2016, Mapzen -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the {organization} nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/walk/Makefile b/vendor/github.com/whosonfirst/walk/Makefile deleted file mode 100644 index 063c2b8..0000000 --- a/vendor/github.com/whosonfirst/walk/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -fmt: - go fmt *.go diff --git a/vendor/github.com/whosonfirst/walk/README.md b/vendor/github.com/whosonfirst/walk/README.md deleted file mode 100644 index ebaf4c4..0000000 --- a/vendor/github.com/whosonfirst/walk/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# walk - -This is a fork of [MichaelTJones](https://github.com/MichaelTJones)' original [walk](https://github.com/MichaelTJones/walk) package. - -## The original version - -_This is what [MichaelTJones](https://github.com/MichaelTJones) wrote in his original [README.md](https://github.com/MichaelTJones/walk/blob/master/README.md) file_: - -Fast parallel version of golang filepath.Walk() - -Performs traversals in parallel so set GOMAXPROCS appropriately. Vaues of 8 to 16 seem to work best on my -4-CPU plus 4 SMT pseudo-CPU MacBookPro. The result is about 4x-6x the traversal rate of the standard Walk(). -The two are not identical since we are walking the file system in a tumult of asynchronous walkFunc calls by -a number of goroutines. So, take note of the following: - -1. This walk honors all of the walkFunc error semantics but as multiple user-supplied walkFuncs may simultaneously encounter a traversal error or generate one to stop traversal, only the FIRST of these will be returned as the Walk() result. - -2. Further, since there may be a few files in flight at the instant of error discovery, a few more walkFunc calls may happen after the first error-generating call has signaled its desire to stop. In general this is a non-issue but it could matter so pay attention when designing your walkFunc. (For example, if you accumulate results then you need to have your own means to know to stop accumulating once you signal an error.) - -3. Because the walkFunc is called concurrently in multiple goroutines, it needs to be careful about what it does with external data to avoid collisions. Results may be printed using fmt, but generally the best plan is to send results over a channel or accumulate counts using a locked mutex. - -These issues are illustrated/handled in the simple traversal programs supplied with walk. There is also a test file that is just the tests from filepath in the Go language's standard library. Walk passes these tests when run in single process mode, and passes most of them in concurrent mode (GOMAXPROCS > 1). The problem is not a real problem, but one of the test expecting a specific number of errors to be found based on presumed sequential traversals. - -## The changes - -### Set number of walkers from runtime.GOMAXPROCS - -This package incorporates [avleen](https://github.com/avleen)'s [fork](https://github.com/avleen/walk) of the walk package to [set the number of walkers from runtime.GOMAXPROCS ](https://github.com/MichaelTJones/walk/compare/master...avleen:master). - -### walk.WalkWithNFSKludge - -This introduces a new package function called `WalkWithNFSKludge` that will trap and ignore `readdirent: errno 523` errors which can occur when traversing NFS mounts. You should use this function with caution and your eyes wide open. - -There is _nothing_ magic happening here. It is a leap of faith that the error in question, which is raised by the operating system, is not really a big deal for the purposes of your application and shouldn't yield a fatal error by the `walk` package. - -File under: 🙈 diff --git a/vendor/github.com/whosonfirst/walk/path_plan9.go b/vendor/github.com/whosonfirst/walk/path_plan9.go deleted file mode 100644 index 86d4e0c..0000000 --- a/vendor/github.com/whosonfirst/walk/path_plan9.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package walk - -import "strings" - -// IsAbs returns true if the path is absolute. -func IsAbs(path string) bool { - return strings.HasPrefix(path, "/") || strings.HasPrefix(path, "#") -} - -// volumeNameLen returns length of the leading volume name on Windows. -// It returns 0 elsewhere. -func volumeNameLen(path string) int { - return 0 -} - -// HasPrefix exists for historical compatibility and should not be used. -func HasPrefix(p, prefix string) bool { - return strings.HasPrefix(p, prefix) -} - -func splitList(path string) []string { - if path == "" { - return []string{} - } - return strings.Split(path, string(ListSeparator)) -} diff --git a/vendor/github.com/whosonfirst/walk/path_unix.go b/vendor/github.com/whosonfirst/walk/path_unix.go deleted file mode 100644 index 24f7343..0000000 --- a/vendor/github.com/whosonfirst/walk/path_unix.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris - -package walk - -import "strings" - -// IsAbs returns true if the path is absolute. -func IsAbs(path string) bool { - return strings.HasPrefix(path, "/") -} - -// volumeNameLen returns length of the leading volume name on Windows. -// It returns 0 elsewhere. -func volumeNameLen(path string) int { - return 0 -} - -// HasPrefix exists for historical compatibility and should not be used. -func HasPrefix(p, prefix string) bool { - return strings.HasPrefix(p, prefix) -} - -func splitList(path string) []string { - if path == "" { - return []string{} - } - return strings.Split(path, string(ListSeparator)) -} diff --git a/vendor/github.com/whosonfirst/walk/path_windows.go b/vendor/github.com/whosonfirst/walk/path_windows.go deleted file mode 100644 index d497a65..0000000 --- a/vendor/github.com/whosonfirst/walk/path_windows.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package walk - -import ( - "strings" -) - -func isSlash(c uint8) bool { - return c == '\\' || c == '/' -} - -// IsAbs returns true if the path is absolute. -func IsAbs(path string) (b bool) { - l := volumeNameLen(path) - if l == 0 { - return false - } - path = path[l:] - if path == "" { - return false - } - return isSlash(path[0]) -} - -// volumeNameLen returns length of the leading volume name on Windows. -// It returns 0 elsewhere. -func volumeNameLen(path string) int { - if len(path) < 2 { - return 0 - } - // with drive letter - c := path[0] - if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { - return 2 - } - // is it UNC - if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && - !isSlash(path[2]) && path[2] != '.' { - // first, leading `\\` and next shouldn't be `\`. its server name. - for n := 3; n < l-1; n++ { - // second, next '\' shouldn't be repeated. - if isSlash(path[n]) { - n++ - // third, following something characters. its share name. - if !isSlash(path[n]) { - if path[n] == '.' { - break - } - for ; n < l; n++ { - if isSlash(path[n]) { - break - } - } - return n - } - break - } - } - } - return 0 -} - -// HasPrefix exists for historical compatibility and should not be used. -func HasPrefix(p, prefix string) bool { - if strings.HasPrefix(p, prefix) { - return true - } - return strings.HasPrefix(strings.ToLower(p), strings.ToLower(prefix)) -} - -func splitList(path string) []string { - // The same implementation is used in LookPath in os/exec; - // consider changing os/exec when changing this. - - if path == "" { - return []string{} - } - - // Split path, respecting but preserving quotes. - list := []string{} - start := 0 - quo := false - for i := 0; i < len(path); i++ { - switch c := path[i]; { - case c == '"': - quo = !quo - case c == ListSeparator && !quo: - list = append(list, path[start:i]) - start = i + 1 - } - } - list = append(list, path[start:]) - - // Remove quotes. - for i, s := range list { - if strings.Contains(s, `"`) { - list[i] = strings.Replace(s, `"`, ``, -1) - } - } - - return list -} diff --git a/vendor/github.com/whosonfirst/walk/symlink.go b/vendor/github.com/whosonfirst/walk/symlink.go deleted file mode 100644 index 2a5fae5..0000000 --- a/vendor/github.com/whosonfirst/walk/symlink.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !windows - -package walk - -import ( - "bytes" - "errors" - "os" - "strings" -) - -func evalSymlinks(path string) (string, error) { - const maxIter = 255 - originalPath := path - // consume path by taking each frontmost path element, - // expanding it if it's a symlink, and appending it to b - var b bytes.Buffer - for n := 0; path != ""; n++ { - if n > maxIter { - return "", errors.New("EvalSymlinks: too many links in " + originalPath) - } - - // find next path component, p - i := strings.IndexRune(path, Separator) - var p string - if i == -1 { - p, path = path, "" - } else { - p, path = path[:i], path[i+1:] - } - - if p == "" { - if b.Len() == 0 { - // must be absolute path - b.WriteRune(Separator) - } - continue - } - - fi, err := os.Lstat(b.String() + p) - if err != nil { - return "", err - } - if fi.Mode()&os.ModeSymlink == 0 { - b.WriteString(p) - if path != "" { - b.WriteRune(Separator) - } - continue - } - - // it's a symlink, put it at the front of path - dest, err := os.Readlink(b.String() + p) - if err != nil { - return "", err - } - if IsAbs(dest) { - b.Reset() - } - path = dest + string(Separator) + path - } - return Clean(b.String()), nil -} diff --git a/vendor/github.com/whosonfirst/walk/symlink_windows.go b/vendor/github.com/whosonfirst/walk/symlink_windows.go deleted file mode 100644 index 389a615..0000000 --- a/vendor/github.com/whosonfirst/walk/symlink_windows.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package walk - -import ( - "syscall" -) - -func toShort(path string) (string, error) { - p, err := syscall.UTF16FromString(path) - if err != nil { - return "", err - } - b := p // GetShortPathName says we can reuse buffer - n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - if n > uint32(len(b)) { - b = make([]uint16, n) - n, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - } - return syscall.UTF16ToString(b), nil -} - -func toLong(path string) (string, error) { - p, err := syscall.UTF16FromString(path) - if err != nil { - return "", err - } - b := p // GetLongPathName says we can reuse buffer - n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - if n > uint32(len(b)) { - b = make([]uint16, n) - n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b))) - if err != nil { - return "", err - } - } - b = b[:n] - return syscall.UTF16ToString(b), nil -} - -func evalSymlinks(path string) (string, error) { - p, err := toShort(path) - if err != nil { - return "", err - } - p, err = toLong(p) - if err != nil { - return "", err - } - // syscall.GetLongPathName does not change the case of the drive letter, - // but the result of EvalSymlinks must be unique, so we have - // EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`). - // Make drive letter upper case. - if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' { - p = string(p[0]+'A'-'a') + p[1:] - } - return Clean(p), nil -} diff --git a/vendor/github.com/whosonfirst/walk/walk.go b/vendor/github.com/whosonfirst/walk/walk.go deleted file mode 100644 index 6d5271f..0000000 --- a/vendor/github.com/whosonfirst/walk/walk.go +++ /dev/null @@ -1,480 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package filepath implements utility routines for manipulating filename paths -// in a way compatible with the target operating system-defined file paths. -package walk - -import ( - "errors" - "log" - "os" - "runtime" - "sort" - "strings" - "sync" -) - -// SkipDir is used as a return value from WalkFuncs to indicate that -// the directory named in the call is to be skipped. It is not returned -// as an error by any function. -var SkipDir = errors.New("skip this directory") - -// WalkFunc is the type of the function called for each file or directory -// visited by Walk. The path argument contains the argument to Walk as a -// prefix; that is, if Walk is called with "dir", which is a directory -// containing the file "a", the walk function will be called with argument -// "dir/a". The info argument is the os.FileInfo for the named path. -// -// If there was a problem walking to the file or directory named by path, the -// incoming error will describe the problem and the function can decide how -// to handle that error (and Walk will not descend into that directory). If -// an error is returned, processing stops. The sole exception is that if path -// is a directory and the function returns the special value SkipDir, the -// contents of the directory are skipped and processing continues as usual on -// the next file. -type WalkFunc func(path string, info os.FileInfo, err error) error - -var lstat = os.Lstat // for testing -var LstatP = &lstat - -type VisitData struct { - path string - info os.FileInfo -} - -type WalkState struct { - walkFn WalkFunc - v chan VisitData // files to be processed - active sync.WaitGroup // number of files to process - lock sync.RWMutex - firstError error // accessed using lock - ignore_errno523 bool -} - -func (ws *WalkState) terminated() bool { - ws.lock.RLock() - done := ws.firstError != nil - ws.lock.RUnlock() - return done -} - -func (ws *WalkState) setTerminated(err error) { - ws.lock.Lock() - if ws.firstError == nil { - ws.firstError = err - } - ws.lock.Unlock() - return -} - -func (ws *WalkState) visitChannel() { - for file := range ws.v { - ws.visitFile(file) - ws.active.Add(-1) - } -} - -func (ws *WalkState) visitFile(file VisitData) { - if ws.terminated() { - return - } - - err := ws.walkFn(file.path, file.info, nil) - if err != nil { - if !(file.info.IsDir() && err == SkipDir) { - ws.setTerminated(err) - } - return - } - - if !file.info.IsDir() { - return - } - - names, err := readDirNames(file.path, ws.ignore_errno523) - if err != nil { - err = ws.walkFn(file.path, file.info, err) - if err != nil { - ws.setTerminated(err) - } - return - } - - here := file.path - for _, name := range names { - file.path = Join(here, name) - file.info, err = lstat(file.path) - if err != nil { - err = ws.walkFn(file.path, file.info, err) - if err != nil && (!file.info.IsDir() || err != SkipDir) { - ws.setTerminated(err) - return - } - } else { - switch file.info.IsDir() { - case true: - ws.active.Add(1) // presume channel send will succeed - select { - case ws.v <- file: - // push directory info to queue for concurrent traversal - default: - // undo increment when send fails and handle now - ws.active.Add(-1) - ws.visitFile(file) - } - case false: - err = ws.walkFn(file.path, file.info, nil) - if err != nil { - ws.setTerminated(err) - return - } - } - } - } -} - -// Walk walks the file tree rooted at root, calling walkFn for each file or -// directory in the tree, including root. All errors that arise visiting files -// and directories are filtered by walkFn. The files are walked in a random -// order. Walk does not follow symbolic links. - -func Walk(root string, walkFn WalkFunc) error { - - return walk(root, walkFn, false) -} - -func WalkWithNFSKludge(root string, walkFn WalkFunc) error { - - return walk(root, walkFn, true) -} - -func walk(root string, walkFn WalkFunc, nfs_kludge bool) error { - - info, err := os.Lstat(root) - - if err != nil { - return walkFn(root, nil, err) - } - - ignore_errno523 := false - - if nfs_kludge { - ignore_errno523 = true - } - - ws := &WalkState{ - walkFn: walkFn, - v: make(chan VisitData, 1024), - ignore_errno523: ignore_errno523, - } - defer close(ws.v) - - ws.active.Add(1) - ws.v <- VisitData{root, info} - - walkers := runtime.GOMAXPROCS(0) - - for i := 0; i < walkers; i++ { - go ws.visitChannel() - } - - ws.active.Wait() - - return ws.firstError -} - -// readDirNames reads the directory named by dirname and returns -// a sorted list of directory entries. -func readDirNames(dirname string, ignore_errno523 bool) ([]string, error) { - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - names, err := f.Readdirnames(-1) - f.Close() - if err != nil { - - // https://forums.aws.amazon.com/thread.jspa?threadID=233852 - - if err.Error() == "readdirent: errno 523" && ignore_errno523 { - log.Printf("got a 523 error for %s, but ignoring\n", dirname) - - if names == nil { - log.Printf("but seriously... %s is empty, which is maybe weird?\n", dirname) - names = make([]string, 0) - } - } else { - return nil, err - } - } - sort.Strings(names) // omit sort to save 1-2% - return names, nil -} - -// -// THE REMAINDER IS UNCHANGED FROM THE ORGINAL GO LIBRARY ORIGINAL -// - -// A lazybuf is a lazily constructed path buffer. -// It supports append, reading previously appended bytes, -// and retrieving the final string. It does not allocate a buffer -// to hold the output until that output diverges from s. -type lazybuf struct { - path string - buf []byte - w int - volAndPath string - volLen int -} - -func (b *lazybuf) index(i int) byte { - if b.buf != nil { - return b.buf[i] - } - return b.path[i] -} - -func (b *lazybuf) append(c byte) { - if b.buf == nil { - if b.w < len(b.path) && b.path[b.w] == c { - b.w++ - return - } - b.buf = make([]byte, len(b.path)) - copy(b.buf, b.path[:b.w]) - } - b.buf[b.w] = c - b.w++ -} - -func (b *lazybuf) string() string { - if b.buf == nil { - return b.volAndPath[:b.volLen+b.w] - } - return b.volAndPath[:b.volLen] + string(b.buf[:b.w]) -} - -const ( - Separator = os.PathSeparator - ListSeparator = os.PathListSeparator -) - -// Clean returns the shortest path name equivalent to path -// by purely lexical processing. It applies the following rules -// iteratively until no further processing can be done: -// -// 1. Replace multiple Separator elements with a single one. -// 2. Eliminate each . path name element (the current directory). -// 3. Eliminate each inner .. path name element (the parent directory) -// along with the non-.. element that precedes it. -// 4. Eliminate .. elements that begin a rooted path: -// that is, replace "/.." by "/" at the beginning of a path, -// assuming Separator is '/'. -// -// The returned path ends in a slash only if it represents a root directory, -// such as "/" on Unix or `C:\` on Windows. -// -// If the result of this process is an empty string, Clean -// returns the string ".". -// -// See also Rob Pike, ``Lexical File Names in Plan 9 or -// Getting Dot-Dot Right,'' -// http://plan9.bell-labs.com/sys/doc/lexnames.html -func Clean(path string) string { - originalPath := path - volLen := volumeNameLen(path) - path = path[volLen:] - if path == "" { - if volLen > 1 && originalPath[1] != ':' { - // should be UNC - return FromSlash(originalPath) - } - return originalPath + "." - } - rooted := os.IsPathSeparator(path[0]) - - // Invariants: - // reading from path; r is index of next byte to process. - // writing to buf; w is index of next byte to write. - // dotdot is index in buf where .. must stop, either because - // it is the leading slash or it is a leading ../../.. prefix. - n := len(path) - out := lazybuf{path: path, volAndPath: originalPath, volLen: volLen} - r, dotdot := 0, 0 - if rooted { - out.append(Separator) - r, dotdot = 1, 1 - } - - for r < n { - switch { - case os.IsPathSeparator(path[r]): - // empty path element - r++ - case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): - // . element - r++ - case path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): - // .. element: remove to last separator - r += 2 - switch { - case out.w > dotdot: - // can backtrack - out.w-- - for out.w > dotdot && !os.IsPathSeparator(out.index(out.w)) { - out.w-- - } - case !rooted: - // cannot backtrack, but not rooted, so append .. element. - if out.w > 0 { - out.append(Separator) - } - out.append('.') - out.append('.') - dotdot = out.w - } - default: - // real path element. - // add slash if needed - if rooted && out.w != 1 || !rooted && out.w != 0 { - out.append(Separator) - } - // copy element - for ; r < n && !os.IsPathSeparator(path[r]); r++ { - out.append(path[r]) - } - } - } - - // Turn empty string into "." - if out.w == 0 { - out.append('.') - } - - return FromSlash(out.string()) -} - -// ToSlash returns the result of replacing each separator character -// in path with a slash ('/') character. Multiple separators are -// replaced by multiple slashes. -func ToSlash(path string) string { - if Separator == '/' { - return path - } - return strings.Replace(path, string(Separator), "/", -1) -} - -// FromSlash returns the result of replacing each slash ('/') character -// in path with a separator character. Multiple slashes are replaced -// by multiple separators. -func FromSlash(path string) string { - if Separator == '/' { - return path - } - return strings.Replace(path, "/", string(Separator), -1) -} - -// Join joins any number of path elements into a single path, adding -// a Separator if necessary. The result is Cleaned, in particular -// all empty strings are ignored. -func Join(elem ...string) string { - for i, e := range elem { - if e != "" { - return Clean(strings.Join(elem[i:], string(Separator))) - } - } - return "" -} - -// Rel returns a relative path that is lexically equivalent to targpath when -// joined to basepath with an intervening separator. That is, -// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself. -// On success, the returned path will always be relative to basepath, -// even if basepath and targpath share no elements. -// An error is returned if targpath can't be made relative to basepath or if -// knowing the current working directory would be necessary to compute it. -func Rel(basepath, targpath string) (string, error) { - baseVol := VolumeName(basepath) - targVol := VolumeName(targpath) - base := Clean(basepath) - targ := Clean(targpath) - if targ == base { - return ".", nil - } - base = base[len(baseVol):] - targ = targ[len(targVol):] - if base == "." { - base = "" - } - // Can't use IsAbs - `\a` and `a` are both relative in Windows. - baseSlashed := len(base) > 0 && base[0] == Separator - targSlashed := len(targ) > 0 && targ[0] == Separator - if baseSlashed != targSlashed || baseVol != targVol { - return "", errors.New("Rel: can't make " + targ + " relative to " + base) - } - // Position base[b0:bi] and targ[t0:ti] at the first differing elements. - bl := len(base) - tl := len(targ) - var b0, bi, t0, ti int - for { - for bi < bl && base[bi] != Separator { - bi++ - } - for ti < tl && targ[ti] != Separator { - ti++ - } - if targ[t0:ti] != base[b0:bi] { - break - } - if bi < bl { - bi++ - } - if ti < tl { - ti++ - } - b0 = bi - t0 = ti - } - if base[b0:bi] == ".." { - return "", errors.New("Rel: can't make " + targ + " relative to " + base) - } - if b0 != bl { - // Base elements left. Must go up before going down. - seps := strings.Count(base[b0:bl], string(Separator)) - size := 2 + seps*3 - if tl != t0 { - size += 1 + tl - t0 - } - buf := make([]byte, size) - n := copy(buf, "..") - for i := 0; i < seps; i++ { - buf[n] = Separator - copy(buf[n+1:], "..") - n += 3 - } - if t0 != tl { - buf[n] = Separator - copy(buf[n+1:], targ[t0:]) - } - return string(buf), nil - } - return targ[t0:], nil -} - -// VolumeName returns leading volume name. -// Given "C:\foo\bar" it returns "C:" under windows. -// Given "\\host\share\foo" it returns "\\host\share". -// On other platforms it returns "". -func VolumeName(path string) (v string) { - return path[:volumeNameLen(path)] -} - -// EvalSymlinks returns the path name after the evaluation of any symbolic -// links. -// If path is relative the result will be relative to the current directory, -// unless one of the components is an absolute symbolic link. -func EvalSymlinks(path string) (string, error) { - return evalSymlinks(path) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index ab43c37..1c91080 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -10,9 +10,6 @@ github.com/aaronland/go-artisanal-integers-proxy/service # github.com/aaronland/go-brooklynintegers-api v1.2.2 ## explicit; go 1.12 github.com/aaronland/go-brooklynintegers-api -# github.com/aaronland/go-json-query v0.0.2 -## explicit; go 1.12 -github.com/aaronland/go-json-query # github.com/aaronland/go-londonintegers-api v0.1.1 ## explicit; go 1.12 github.com/aaronland/go-londonintegers-api @@ -58,6 +55,9 @@ github.com/paulmach/orb github.com/paulmach/orb/geojson github.com/paulmach/orb/internal/length github.com/paulmach/orb/planar +# github.com/saracen/walker v0.1.2 +## explicit; go 1.12 +github.com/saracen/walker # github.com/sfomuseum/go-edtf v0.2.3 ## explicit; go 1.12 github.com/sfomuseum/go-edtf @@ -87,9 +87,6 @@ github.com/tidwall/pretty # github.com/tidwall/sjson v1.1.7 ## explicit; go 1.14 github.com/tidwall/sjson -# github.com/whosonfirst/go-whosonfirst-crawl v0.2.1 -## explicit; go 1.12 -github.com/whosonfirst/go-whosonfirst-crawl # github.com/whosonfirst/go-whosonfirst-export v0.4.0 ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-export @@ -119,10 +116,6 @@ github.com/whosonfirst/go-whosonfirst-id # github.com/whosonfirst/go-whosonfirst-id-proxy v0.0.1 ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-id-proxy/provider -# github.com/whosonfirst/go-whosonfirst-index v0.3.4 -## explicit; go 1.12 -github.com/whosonfirst/go-whosonfirst-index -github.com/whosonfirst/go-whosonfirst-index/fs # github.com/whosonfirst/go-whosonfirst-log v0.1.0 ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-log @@ -134,17 +127,12 @@ github.com/whosonfirst/go-whosonfirst-placetypes/placetypes ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-sources github.com/whosonfirst/go-whosonfirst-sources/sources -# github.com/whosonfirst/go-whosonfirst-spr v0.1.0 -## explicit; go 1.12 # github.com/whosonfirst/go-whosonfirst-spr/v2 v2.0.0 ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-spr/v2 # github.com/whosonfirst/go-whosonfirst-uri v1.1.0 ## explicit; go 1.12 github.com/whosonfirst/go-whosonfirst-uri -# github.com/whosonfirst/walk v0.0.1 -## explicit; go 1.12 -github.com/whosonfirst/walk # github.com/whosonfirst/warning v0.1.1 ## explicit; go 1.12 github.com/whosonfirst/warning diff --git a/wofdata/wofdata.go b/wofdata/wofdata.go index e2f75c3..06f0e2d 100644 --- a/wofdata/wofdata.go +++ b/wofdata/wofdata.go @@ -3,10 +3,8 @@ package wofdata import ( "bufio" "bytes" - "context" "errors" "fmt" - "io" "log" "os" "path/filepath" @@ -20,15 +18,12 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/sjson" + "github.com/saracen/walker" export "github.com/whosonfirst/go-whosonfirst-export" "github.com/whosonfirst/go-whosonfirst-export/options" geojson "github.com/whosonfirst/go-whosonfirst-geojson-v2" "github.com/whosonfirst/go-whosonfirst-geojson-v2/feature" - index "github.com/whosonfirst/go-whosonfirst-index" uri "github.com/whosonfirst/go-whosonfirst-uri" - - // Enable the filesystem driver - _ "github.com/whosonfirst/go-whosonfirst-index/fs" ) type WOFData struct { @@ -44,10 +39,13 @@ func NewWOFData(dataPath string, expOpts options.Options) *WOFData { // Iterate fires the provided callback for every file in the WOFData path. func (d *WOFData) Iterate(cb func(geojson.Feature) error) error { - localCb := func(ctx context.Context, fh io.Reader, args ...interface{}) error { - path, err := index.PathForContext(ctx) - if err != nil { - return err + walkFn := func(path string, fi os.FileInfo) error { + if fi.IsDir() { + return nil + } + + if !strings.HasSuffix(path, ".geojson") { + return nil } f, err := feature.LoadWOFFeatureFromFile(path) @@ -55,22 +53,14 @@ func (d *WOFData) Iterate(cb func(geojson.Feature) error) error { return err } - err = cb(f) - - return err + return cb(f) } - i, err := index.NewIndexer("directory", localCb) - if err != nil { - return err - } - - err = i.IndexPath(d.dataPath) - if err != nil { + errorFn := walker.WithErrorCallback(func(path string, err error) error { return err - } + }) - return nil + return walker.Walk(d.dataPath, walkFn, errorFn) } const edtfDateLayout = "2006-01-02"