diff --git a/build/pbf2json.darwin-x64 b/build/pbf2json.darwin-x64 index d86fd04..54acb16 100755 Binary files a/build/pbf2json.darwin-x64 and b/build/pbf2json.darwin-x64 differ diff --git a/build/pbf2json.linux-arm b/build/pbf2json.linux-arm index 2e4d91a..57bb05e 100755 Binary files a/build/pbf2json.linux-arm and b/build/pbf2json.linux-arm differ diff --git a/build/pbf2json.linux-x64 b/build/pbf2json.linux-x64 index efb0416..d45f649 100755 Binary files a/build/pbf2json.linux-x64 and b/build/pbf2json.linux-x64 differ diff --git a/build/pbf2json.win32-x64 b/build/pbf2json.win32-x64 index 9dddb53..6ffd202 100755 Binary files a/build/pbf2json.win32-x64 and b/build/pbf2json.win32-x64 differ diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..e36f621 --- /dev/null +++ b/cache.go @@ -0,0 +1,87 @@ +package main + +import ( + "github.com/qedus/osmpbf" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "log" + "strconv" +) + +// queue a leveldb write in a batch +func cacheQueueNode(batch *leveldb.Batch, node *osmpbf.Node) { + id, val := nodeToBytes(node) + batch.Put([]byte(id), []byte(val)) +} + +// queue a leveldb write in a batch +func cacheQueueWay(batch *leveldb.Batch, way *osmpbf.Way) { + id, val := wayToBytes(way) + batch.Put([]byte(id), []byte(val)) +} + +// flush a leveldb batch to database and reset batch to 0 +func cacheFlush(db *leveldb.DB, batch *leveldb.Batch, sync bool) { + var writeOpts = &opt.WriteOptions{ + NoWriteMerge: true, + Sync: sync, + } + + err := db.Write(batch, writeOpts) + if err != nil { + log.Fatal(err) + } + batch.Reset() +} + +func cacheLookupNodeByID(db *leveldb.DB, id int64) (map[string]string, error) { + stringid := strconv.FormatInt(id, 10) + + data, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] fetch failed for node ID:", stringid) + return make(map[string]string, 0), err + } + + return bytesToLatLon(data), nil +} + +func cacheLookupNodes(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) { + + var container []map[string]string + + for _, each := range way.NodeIDs { + stringid := strconv.FormatInt(each, 10) + + data, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] denormalize failed for way:", way.ID, "node not found:", stringid) + return make([]map[string]string, 0), err + } + + container = append(container, bytesToLatLon(data)) + } + + return container, nil +} + +func cacheLookupWayNodes(db *leveldb.DB, wayid int64) ([]map[string]string, error) { + + // prefix the key with 'W' to differentiate it from node ids + stringid := "W" + strconv.FormatInt(wayid, 10) + + // look up way bytes + reldata, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] lookup failed for way:", wayid, "noderefs not found:", stringid) + return make([]map[string]string, 0), err + } + + // generate a way object + var way = &osmpbf.Way{ + ID: wayid, + NodeIDs: bytesToIDSlice(reldata), + } + + return cacheLookupNodes(db, way) +} diff --git a/pbf2json.go b/pbf2json.go index a09f2ee..b06533c 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -16,7 +16,6 @@ import ( geo "github.com/paulmach/go.geo" "github.com/qedus/osmpbf" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" ) type settings struct { @@ -495,84 +494,6 @@ func isWheelchairAccessibleNode(node *osmpbf.Node) uint8 { return 0 } -// queue a leveldb write in a batch -func cacheQueueNode(batch *leveldb.Batch, node *osmpbf.Node) { - id, val := nodeToBytes(node) - batch.Put([]byte(id), []byte(val)) -} - -// queue a leveldb write in a batch -func cacheQueueWay(batch *leveldb.Batch, way *osmpbf.Way) { - id, val := wayToBytes(way) - batch.Put([]byte(id), []byte(val)) -} - -// flush a leveldb batch to database and reset batch to 0 -func cacheFlush(db *leveldb.DB, batch *leveldb.Batch, sync bool) { - var writeOpts = &opt.WriteOptions{ - NoWriteMerge: true, - Sync: sync, - } - - err := db.Write(batch, writeOpts) - if err != nil { - log.Fatal(err) - } - batch.Reset() -} - -func cacheLookupNodeByID(db *leveldb.DB, id int64) (map[string]string, error) { - stringid := strconv.FormatInt(id, 10) - - data, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] fetch failed for node ID:", stringid) - return make(map[string]string, 0), err - } - - return bytesToLatLon(data), nil -} - -func cacheLookupNodes(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) { - - var container []map[string]string - - for _, each := range way.NodeIDs { - stringid := strconv.FormatInt(each, 10) - - data, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] denormalize failed for way:", way.ID, "node not found:", stringid) - return make([]map[string]string, 0), err - } - - container = append(container, bytesToLatLon(data)) - } - - return container, nil -} - -func cacheLookupWayNodes(db *leveldb.DB, wayid int64) ([]map[string]string, error) { - - // prefix the key with 'W' to differentiate it from node ids - stringid := "W" + strconv.FormatInt(wayid, 10) - - // look up way bytes - reldata, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] lookup failed for way:", wayid, "noderefs not found:", stringid) - return make([]map[string]string, 0), err - } - - // generate a way object - var way = &osmpbf.Way{ - ID: wayid, - NodeIDs: bytesToIDSlice(reldata), - } - - return cacheLookupNodes(db, way) -} - // decode bytes to a 'latlon' type object func bytesToLatLon(data []byte) map[string]string { buf := make([]byte, 8)