Skip to content

Commit

Permalink
Merge pull request #3 from nextmv-io/revert-2-deflate
Browse files Browse the repository at this point in the history 
Revert "replace czlib with go-libdeflate"
  • Loading branch information
@larsbeck
larsbeck authored Nov 15, 2023
2 parents 6a17f88 + bb1b92c commit 34f6872
Show file tree
Hide file tree
Showing 8 changed files with 69 additions and 163 deletions.
15 changes: 8 additions & 7 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,28 @@ jobs:
build-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v2

- name: Setup Go
uses: actions/setup-go@v4
uses: actions/setup-go@v2
with:
go-version: '1.19.13'
cache: false
go-version: '1.15.3'

- name: Install dependencies
run: |
go version
go get -u golang.org/x/lint/golint
- name: Run build
run: go build .

- name: Run tests
run: go test -v -coverprofile=profile.cov ./...

- name: Run vet & lint
run: |
go vet .
golint .
- name: Run tests
run: go test -v -coverprofile=profile.cov ./...

- name: codecov
uses: codecov/codecov-action@v1
Expand Down
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,9 @@ BenchmarkChange_UnmarshalJSON-12 287707 317723 +10.43%
## CGO and zlib

OSM PBF data comes in blocks, each block is zlib compressed. Decompressing this
data takes about 33% of the total read time. [4kills/go-libdeflate](https://github.com/4kills/go-libdeflate) is
used to speed up decompressing.
See [osmpbf/README.md](osmpbf#using-libdeflate-for-decompression) for more details.
data takes about 33% of the total read time. [DataDog/czlib](https://github.com/DataDog/czlib) is
used to speed this process.
See [osmpbf/README.md](osmpbf#using-cgoczlib-for-decompression) for more details.

As a result, a C compiler is necessary to install this module. On macOS this may require
installing pkg-config using something like `brew install pkg-config`
Expand Down
9 changes: 4 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
module github.com/nextmv-io/osm

go 1.19
go 1.13

require (
github.com/4kills/go-libdeflate/v2 v2.0.3
github.com/paulmach/orb v0.10.0
github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2
github.com/paulmach/orb v0.1.3
github.com/paulmach/protoscan v0.2.1
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.27.1
)

require go.mongodb.org/mongo-driver v1.12.1 // indirect
85 changes: 4 additions & 81 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,94 +1,17 @@
github.com/4kills/go-libdeflate/v2 v2.0.3 h1:Y13oRUvtAXFJkcW4F0MnaQQB753a71sTutGrVbEAubQ=
github.com/4kills/go-libdeflate/v2 v2.0.3/go.mod h1:hyouZv4OAhHaaMpYuejstUN0xOg8mA+yy75WE3Ty6SM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo=
github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s=
github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8=
github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
48 changes: 20 additions & 28 deletions osmpbf/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,39 +80,31 @@ This package supports reading OSM PBF files where the ways have been annotated w

Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html).

## Using cgo libdeflate for decompression
## Using cgo/czlib for decompression

OSM PBF files are a set of blocks that are zlib compressed. When using the pure golang
implementation this can account for about 1/3 of the read time. When cgo is enabled
the package [go-libdeflate](https://github.com/4kills/libdeflate) will used.

Previous versions used the lib czlib based on zlib. libdeflate is more performant
and more memory efficient for uncompressing.
the package will used [czlib](https://github.com/DataDog/czlib).

```
$ CGO_ENABLED=0 go test -bench . > disabled.txt
$ CGO_ENABLED=1 go test -bench . > enabled.txt
$ benchcmp disabled.txt enabled.txt
benchmark old ns/op new ns/op delta
BenchmarkLondon-8 361519289 275254714 -23.86%
BenchmarkLondon_withFiltersTrue-8 392469042 263935960 -32.75%
BenchmarkLondon_withFiltersFalse-8 310824940 200477972 -35.50%
BenchmarkLondon_nodes-8 295277528 180614979 -38.83%
BenchmarkLondon_ways-8 257494509 140700970 -45.36%
BenchmarkLondon_relations-8 189490128 75263200 -60.28%
benchmark old allocs new allocs delta
BenchmarkLondon-8 4863784 4808526 -1.14%
BenchmarkLondon_withFiltersTrue-8 4863786 4808515 -1.14%
BenchmarkLondon_withFiltersFalse-8 1419995 1364724 -3.89%
BenchmarkLondon_nodes-8 3450825 3395559 -1.60%
BenchmarkLondon_ways-8 1851359 1796099 -2.98%
BenchmarkLondon_relations-8 515422 460152 -10.72%
benchmark old bytes new bytes delta
BenchmarkLondon-8 947061317 924789892 -2.35%
BenchmarkLondon_withFiltersTrue-8 947061146 924787588 -2.35%
BenchmarkLondon_withFiltersFalse-8 388725836 366452840 -5.73%
BenchmarkLondon_nodes-8 641663624 619391213 -3.47%
BenchmarkLondon_ways-8 460631859 438360054 -4.84%
BenchmarkLondon_relations-8 206899749 184626277 -10.77%
benchmark old ns/op new ns/op delta
BenchmarkLondon-12 312294630 229927205 -26.37%
BenchmarkLondon_nodes-12 246562457 160021768 -35.10%
BenchmarkLondon_ways-12 216803544 134747327 -37.85%
BenchmarkLondon_relations-12 158722633 80560144 -49.24%
benchmark old allocs new allocs delta
BenchmarkLondon-12 2469128 2416804 -2.12%
BenchmarkLondon_nodes-12 1056166 1003850 -4.95%
BenchmarkLondon_ways-12 1845032 1792716 -2.84%
BenchmarkLondon_relations-12 509090 456772 -10.28%
benchmark old bytes new bytes delta
BenchmarkLondon-12 963734544 954877896 -0.92%
BenchmarkLondon_nodes-12 658337435 649482060 -1.35%
BenchmarkLondon_ways-12 441674734 432819378 -2.00%
BenchmarkLondon_relations-12 187941609 179086389 -4.71%
```
23 changes: 22 additions & 1 deletion osmpbf/decode.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package osmpbf

import (
"bytes"
"context"
"encoding/binary"
"errors"
Expand Down Expand Up @@ -343,8 +344,28 @@ func getData(blob *osmpbf.Blob, data []byte) ([]byte, error) {
return blob.GetRaw(), nil

case blob.ZlibData != nil:
return decompress(blob.GetZlibData(), (int)(blob.GetRawSize()), data)
r, err := zlibReader(blob.GetZlibData())
if err != nil {
return nil, err
}

// using the bytes.Buffer allows for the preallocation of the necessary space.
l := blob.GetRawSize() + bytes.MinRead
if cap(data) < int(l) {
data = make([]byte, 0, l+l/10)
} else {
data = data[:0]
}
buf := bytes.NewBuffer(data)
if _, err = buf.ReadFrom(r); err != nil {
return nil, err
}

if buf.Len() != int(blob.GetRawSize()) {
return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), blob.GetRawSize())
}

return buf.Bytes(), nil
default:
return nil, errors.New("unknown blob data")
}
Expand Down
20 changes: 5 additions & 15 deletions osmpbf/zlib_cgo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,14 @@
package osmpbf

import (
"fmt"
"bytes"
"io"

deflate "github.com/4kills/go-libdeflate/v2"
"github.com/datadog/czlib"
)

func decompress(in []byte, size int, data []byte) ([]byte, error) {
if cap(data) > (int)(size) {
data = data[0:size]
} else {
data = nil
}

_, buf, err := deflate.DecompressZlib(in, data)
if len(buf) != int(size) {
return nil, fmt.Errorf("raw blob data size %d but expected %d", len(buf), size)
}

return buf, err
func zlibReader(data []byte) (io.ReadCloser, error) {
return czlib.NewReader(bytes.NewReader(data))
}

func zlibWriter(w io.Writer) io.WriteCloser {
Expand Down
26 changes: 3 additions & 23 deletions osmpbf/zlib_go.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,11 @@ package osmpbf
import (
"bytes"
"compress/zlib"
"fmt"
"io"
)

func decompress(in []byte, size int, data []byte) ([]byte, error) {
r, err := zlib.NewReader(bytes.NewReader(in))
if err != nil {
return nil, err
}

// using the bytes.Buffer allows for the preallocation of the necessary space.
l := size + bytes.MinRead
if cap(data) < int(l) {
data = make([]byte, 0, l+l/10)
} else {
data = data[:0]
}
buf := bytes.NewBuffer(data)
if _, err = buf.ReadFrom(r); err != nil {
return nil, err
}

if buf.Len() != int(size) {
return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), size)
}
return buf.Bytes(), nil
func zlibReader(data []byte) (io.ReadCloser, error) {
return zlib.NewReader(bytes.NewReader(data))
}

func zlibWriter(w io.Writer) io.WriteCloser {
Expand Down

0 comments on commit 34f6872

Please sign in to comment.