From 43b9b052f0ef648662c017656ff67c46320bdf65 Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Thu, 22 Jul 2021 16:25:52 +0200 Subject: [PATCH 1/7] feat(relation-outer-members-centroid): move LevelDB functions to cache.go --- cache.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++ pbf2json.go | 97 +++++------------------------------------------------ 2 files changed, 96 insertions(+), 88 deletions(-) create mode 100644 cache.go diff --git a/cache.go b/cache.go new file mode 100644 index 0000000..48c4556 --- /dev/null +++ b/cache.go @@ -0,0 +1,87 @@ +package main + +import ( + "github.com/qedus/osmpbf" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/opt" + "log" + "strconv" +) + +// CacheQueueNode queue a leveldb write in a batch +func CacheQueueNode(batch *leveldb.Batch, node *osmpbf.Node) { + id, val := nodeToBytes(node) + batch.Put([]byte(id), []byte(val)) +} + +// CacheQueueWay queue a leveldb write in a batch +func CacheQueueWay(batch *leveldb.Batch, way *osmpbf.Way) { + id, val := wayToBytes(way) + batch.Put([]byte(id), []byte(val)) +} + +// CacheFlush flush a leveldb batch to database and reset batch to 0 +func CacheFlush(db *leveldb.DB, batch *leveldb.Batch, sync bool) { + var writeOpts = &opt.WriteOptions{ + NoWriteMerge: true, + Sync: sync, + } + + err := db.Write(batch, writeOpts) + if err != nil { + log.Fatal(err) + } + batch.Reset() +} + +func CacheLookupNodeByID(db *leveldb.DB, id int64) (map[string]string, error) { + stringid := strconv.FormatInt(id, 10) + + data, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] fetch failed for node ID:", stringid) + return make(map[string]string, 0), err + } + + return bytesToLatLon(data), nil +} + +func CacheLookupNodes(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) { + + var container []map[string]string + + for _, each := range way.NodeIDs { + stringid := strconv.FormatInt(each, 10) + + data, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] denormalize failed for way:", way.ID, "node not found:", stringid) + return make([]map[string]string, 0), err + } + + container = append(container, bytesToLatLon(data)) + } + + return container, nil +} + +func CacheLookupWayNodes(db *leveldb.DB, wayid int64) ([]map[string]string, error) { + + // prefix the key with 'W' to differentiate it from node ids + stringid := "W" + strconv.FormatInt(wayid, 10) + + // look up way bytes + reldata, err := db.Get([]byte(stringid), nil) + if err != nil { + log.Println("[warn] lookup failed for way:", wayid, "noderefs not found:", stringid) + return make([]map[string]string, 0), err + } + + // generate a way object + var way = &osmpbf.Way{ + ID: wayid, + NodeIDs: bytesToIDSlice(reldata), + } + + return CacheLookupNodes(db, way) +} diff --git a/pbf2json.go b/pbf2json.go index a09f2ee..d6eb278 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -16,7 +16,6 @@ import ( geo "github.com/paulmach/go.geo" "github.com/qedus/osmpbf" "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/opt" ) type settings struct { @@ -225,9 +224,9 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if masks.WayRefs.Has(v.ID) || masks.RelNodes.Has(v.ID) { // write in batches - cacheQueueNode(batch, v) + CacheQueueNode(batch, v) if batch.Len() > config.BatchSize { - cacheFlush(db, batch, true) + CacheFlush(db, batch, true) } } @@ -250,7 +249,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if !finishedNodes { finishedNodes = true if batch.Len() > 1 { - cacheFlush(db, batch, true) + CacheFlush(db, batch, true) } } @@ -261,9 +260,9 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if masks.RelWays.Has(v.ID) { // write in batches - cacheQueueWay(batch, v) + CacheQueueWay(batch, v) if batch.Len() > config.BatchSize { - cacheFlush(db, batch, true) + CacheFlush(db, batch, true) } } @@ -272,7 +271,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if masks.Ways.Has(v.ID) { // lookup from leveldb - latlons, err := cacheLookupNodes(db, v) + latlons, err := CacheLookupNodes(db, v) // skip ways which fail to denormalize if err != nil { @@ -302,7 +301,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if !finishedWays { finishedWays = true if batch.Len() > 1 { - cacheFlush(db, batch, true) + CacheFlush(db, batch, true) } } @@ -359,7 +358,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings if v.Tags["boundary"] == "administrative" { for _, member := range v.Members { if member.Type == 0 && member.Role == "admin_centre" { - if latlons, err := cacheLookupNodeByID(db, member.ID); err == nil { + if latlons, err := CacheLookupNodeByID(db, member.ID); err == nil { latlons["type"] = "admin_centre" centroid = latlons break @@ -392,7 +391,7 @@ func findMemberWayLatLons(db *leveldb.DB, v *osmpbf.Relation) [][]map[string]str if mem.Type == 1 { // lookup from leveldb - latlons, err := cacheLookupWayNodes(db, mem.ID) + latlons, err := CacheLookupWayNodes(db, mem.ID) // skip way if it fails to denormalize if err != nil { @@ -495,84 +494,6 @@ func isWheelchairAccessibleNode(node *osmpbf.Node) uint8 { return 0 } -// queue a leveldb write in a batch -func cacheQueueNode(batch *leveldb.Batch, node *osmpbf.Node) { - id, val := nodeToBytes(node) - batch.Put([]byte(id), []byte(val)) -} - -// queue a leveldb write in a batch -func cacheQueueWay(batch *leveldb.Batch, way *osmpbf.Way) { - id, val := wayToBytes(way) - batch.Put([]byte(id), []byte(val)) -} - -// flush a leveldb batch to database and reset batch to 0 -func cacheFlush(db *leveldb.DB, batch *leveldb.Batch, sync bool) { - var writeOpts = &opt.WriteOptions{ - NoWriteMerge: true, - Sync: sync, - } - - err := db.Write(batch, writeOpts) - if err != nil { - log.Fatal(err) - } - batch.Reset() -} - -func cacheLookupNodeByID(db *leveldb.DB, id int64) (map[string]string, error) { - stringid := strconv.FormatInt(id, 10) - - data, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] fetch failed for node ID:", stringid) - return make(map[string]string, 0), err - } - - return bytesToLatLon(data), nil -} - -func cacheLookupNodes(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) { - - var container []map[string]string - - for _, each := range way.NodeIDs { - stringid := strconv.FormatInt(each, 10) - - data, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] denormalize failed for way:", way.ID, "node not found:", stringid) - return make([]map[string]string, 0), err - } - - container = append(container, bytesToLatLon(data)) - } - - return container, nil -} - -func cacheLookupWayNodes(db *leveldb.DB, wayid int64) ([]map[string]string, error) { - - // prefix the key with 'W' to differentiate it from node ids - stringid := "W" + strconv.FormatInt(wayid, 10) - - // look up way bytes - reldata, err := db.Get([]byte(stringid), nil) - if err != nil { - log.Println("[warn] lookup failed for way:", wayid, "noderefs not found:", stringid) - return make([]map[string]string, 0), err - } - - // generate a way object - var way = &osmpbf.Way{ - ID: wayid, - NodeIDs: bytesToIDSlice(reldata), - } - - return cacheLookupNodes(db, way) -} - // decode bytes to a 'latlon' type object func bytesToLatLon(data []byte) map[string]string { buf := make([]byte, 8) From 213adc160a7084b3793011de0e62daff7029817f Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Thu, 22 Jul 2021 16:48:17 +0200 Subject: [PATCH 2/7] feat(relation-outer-members-centroid): create util functions for misc operations in util.go --- pbf2json.go | 8 +++----- util.go | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 util.go diff --git a/pbf2json.go b/pbf2json.go index d6eb278..0be25b3 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -336,7 +336,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings continue } - area := math.Max(wayBounds.GeoWidth(), 0.000001) * math.Max(wayBounds.GeoHeight(), 0.000001) + area := GetAreaOfBounds(wayBounds) // find the way with the largest area if area > largestArea { @@ -713,7 +713,7 @@ func computeCentroidAndBounds(latlons []map[string]string) (map[string]string, * // by comparing first and last coordinates. isClosed := false if points.Length() > 2 { - isClosed = points.First().Equals(points.Last()) + isClosed = IsPointSetClosed(points) } // compute the centroid using one of two different algorithms @@ -725,9 +725,7 @@ func computeCentroidAndBounds(latlons []map[string]string) (map[string]string, * } // return point as lat/lon map - var centroid = make(map[string]string) - centroid["lat"] = strconv.FormatFloat(compute.Lat(), 'f', 7, 64) - centroid["lon"] = strconv.FormatFloat(compute.Lng(), 'f', 7, 64) + centroid := PointToLatLon(compute) return centroid, points.Bound() } diff --git a/util.go b/util.go new file mode 100644 index 0000000..2637f6d --- /dev/null +++ b/util.go @@ -0,0 +1,23 @@ +package main + +import ( + geo "github.com/paulmach/go.geo" + "math" + "strconv" +) + +func IsPointSetClosed(points *geo.PointSet) bool { + return points.First().Equals(points.Last()) +} + +func PointToLatLon(point *geo.Point) map[string]string { + var latLon = make(map[string]string) + latLon["lat"] = strconv.FormatFloat(point.Lat(), 'f', 7, 64) + latLon["lon"] = strconv.FormatFloat(point.Lng(), 'f', 7, 64) + + return latLon +} + +func GetAreaOfBounds(bound *geo.Bound) float64 { + return math.Max(bound.GeoWidth(), 0.000001) * math.Max(bound.GeoHeight(), 0.000001) +} From a3791a59790b9c7b53ec7f7bf60a3946ba404596 Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Fri, 23 Jul 2021 00:18:01 +0200 Subject: [PATCH 3/7] feat(relation-outer-members-centroid): add length check to IsPointSetClosed --- pbf2json.go | 5 +---- util.go | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pbf2json.go b/pbf2json.go index 0be25b3..e861cad 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -711,10 +711,7 @@ func computeCentroidAndBounds(latlons []map[string]string) (map[string]string, * // determine if the way is a closed centroid or a linestring // by comparing first and last coordinates. - isClosed := false - if points.Length() > 2 { - isClosed = IsPointSetClosed(points) - } + isClosed := IsPointSetClosed(points) // compute the centroid using one of two different algorithms var compute *geo.Point diff --git a/util.go b/util.go index 2637f6d..a1da12c 100644 --- a/util.go +++ b/util.go @@ -7,7 +7,10 @@ import ( ) func IsPointSetClosed(points *geo.PointSet) bool { - return points.First().Equals(points.Last()) + if points.Length() > 2 { + return points.First().Equals(points.Last()) + } + return false } func PointToLatLon(point *geo.Point) map[string]string { From ec5f5fd9685c36855d7a0ad7d7a2918c3f754bb7 Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Mon, 9 Aug 2021 18:36:26 +0200 Subject: [PATCH 4/7] feat(relation-outer-members-centroid): change findMemberWayLatLons to return map of members and geo coordinates --- pbf2json.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pbf2json.go b/pbf2json.go index e861cad..aa3ecf0 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -384,22 +384,19 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings } // lookup all latlons for all ways in relation -func findMemberWayLatLons(db *leveldb.DB, v *osmpbf.Relation) [][]map[string]string { - var memberWayLatLons [][]map[string]string +func findMemberWayLatLons(db *leveldb.DB, v *osmpbf.Relation) map[osmpbf.Member][]map[string]string { + var memberWayLatLons = make(map[osmpbf.Member][]map[string]string) for _, mem := range v.Members { - if mem.Type == 1 { + // lookup from leveldb + latlons, err := CacheLookupWayNodes(db, mem.ID) - // lookup from leveldb - latlons, err := CacheLookupWayNodes(db, mem.ID) - - // skip way if it fails to denormalize - if err != nil { - break - } - - memberWayLatLons = append(memberWayLatLons, latlons) + // skip way if it fails to denormalize + if err != nil { + break } + + memberWayLatLons[mem] = latlons } return memberWayLatLons From 1e9741681449fc3807e911f812dbd9dc48963472 Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Mon, 9 Aug 2021 19:26:48 +0200 Subject: [PATCH 5/7] feat(relation-outer-members-centroid): add util function to convert lat long Maps to PointSet --- util.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util.go b/util.go index a1da12c..5bb6034 100644 --- a/util.go +++ b/util.go @@ -21,6 +21,18 @@ func PointToLatLon(point *geo.Point) map[string]string { return latLon } +func LatLngMapToPointSet(latLons []map[string]string) *geo.PointSet { + points := geo.NewPointSet() + + for _, each := range latLons { + var lon, _ = strconv.ParseFloat(each["lon"], 64) + var lat, _ = strconv.ParseFloat(each["lat"], 64) + points.Push(geo.NewPoint(lon, lat)) + } + + return points +} + func GetAreaOfBounds(bound *geo.Bound) float64 { return math.Max(bound.GeoWidth(), 0.000001) * math.Max(bound.GeoHeight(), 0.000001) } From 2168915ec3e8d11e362be8c01b6ddc82188d2be2 Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Tue, 10 Aug 2021 00:53:22 +0200 Subject: [PATCH 6/7] feat(relation-outer-members-centroid): add functionality to compute outer area centroid from non-closed ways --- relation_centroid.go | 221 ++++++++++++++++++++++++++++++++++++++ relation_centroid_test.go | 165 ++++++++++++++++++++++++++++ 2 files changed, 386 insertions(+) create mode 100644 relation_centroid.go create mode 100644 relation_centroid_test.go diff --git a/relation_centroid.go b/relation_centroid.go new file mode 100644 index 0000000..e11f310 --- /dev/null +++ b/relation_centroid.go @@ -0,0 +1,221 @@ +package main + +import ( + geo "github.com/paulmach/go.geo" + "github.com/qedus/osmpbf" + "log" +) + +// ComputeRelationCentroidAndBounds Favors lat, long of member with role admin_centre for centroid, computes outer area if they are non-closed ways +func ComputeRelationCentroidAndBounds(relMemberLatLons map[osmpbf.Member][]map[string]string) (map[string]string, *geo.Bound) { + centroid, isCentroidAdminCentre, bounds, openOuterAreaMemberPoints := getCentroidAndBoundsAndOpenWays(relMemberLatLons) + + if len(openOuterAreaMemberPoints) > 0 { + outerPolygons := getPolygonsFromWays(openOuterAreaMemberPoints) + + if len(outerPolygons) > 0 { + largestArea := 0.0 + + if bounds != nil { + largestArea = GetAreaOfBounds(bounds) + } + + largestOuterPolygon := getLargestAreaPolygon(outerPolygons) + largestOuterPolygonBounds := largestOuterPolygon.Bound() + + if GetAreaOfBounds(largestOuterPolygonBounds) > largestArea { + bounds = largestOuterPolygonBounds + if !isCentroidAdminCentre { + centroidPoint := GetPolygonCentroid(largestOuterPolygon) + centroid = PointToLatLon(centroidPoint) + } + } + } + } + + return centroid, bounds +} + +func getCentroidAndBoundsAndOpenWays(relMemberLatLons map[osmpbf.Member][]map[string]string) (map[string]string, bool, *geo.Bound, []*geo.PointSet) { + openOuterAreaMemberPoints := make([]*geo.PointSet, 0) + foundAdminCentre := false + + var largestArea = 0.0 + var centroid map[string]string + var bounds *geo.Bound + + for member, memberWayLatLons := range relMemberLatLons { + if len(memberWayLatLons) < 1 { + continue + } + + memberWayPointSet := LatLngMapToPointSet(memberWayLatLons) + isClosedWay := IsPointSetClosed(memberWayPointSet) + + if member.Role == "outer" && !isClosedWay { + openOuterAreaMemberPoints = append(openOuterAreaMemberPoints, memberWayPointSet) + } else if member.Type == 0 && member.Role == "admin_centre" { + // prefer admin_center for centroid over computing it + var latlons = memberWayLatLons[0] + + latlons["type"] = "admin_centre" + centroid = latlons + + foundAdminCentre = true + } else { + wayCentroid, wayBounds := ComputeCentroidAndBounds(memberWayLatLons) + + // if for any reason we failed to find a valid bounds + if nil == wayBounds { + log.Println("[warn] failed to calculate bounds for relation member way") + continue + } + + area := GetAreaOfBounds(wayBounds) + + // find the way with the largest area + if area > largestArea { + largestArea = area + if !foundAdminCentre { + centroid = wayCentroid + } + bounds = wayBounds + } + } + } + + return centroid, foundAdminCentre, bounds, openOuterAreaMemberPoints +} + +func getLargestAreaPolygon(polygons []*geo.PointSet) *geo.PointSet { + largestOuterArea := 0.0 + var largestPolygon *geo.PointSet + + for _, polygon := range polygons { + bounds := polygon.Bound() + area := GetAreaOfBounds(bounds) + + if area > largestOuterArea { + largestOuterArea = area + largestPolygon = polygon + } + } + + return largestPolygon +} + +func getClosedWays(ways []*geo.PointSet) []*geo.PointSet { + closedWays := make([]*geo.PointSet, 0) + + for _, way := range ways { + if way.First().Equals(way.Last()) { + closedWays = append(closedWays, way) + } + } + + return closedWays +} + +func getPolygonsFromWays(ways []*geo.PointSet) []*geo.PointSet { + lastIterationOpenWayCount := len(ways) + tryForClockwiseWays := false + + for len(ways) > 1 { + ways = connectWays(ways, tryForClockwiseWays) + + if len(ways) == lastIterationOpenWayCount { + if tryForClockwiseWays { + // no clockwise ways, abort + break + } else { + // no directly connecting ways left, check for clockwise connecting ways + tryForClockwiseWays = true + lastIterationOpenWayCount = 0 + } + } + + lastIterationOpenWayCount = len(ways) + } + + closedWays := getClosedWays(ways) + + return closedWays +} + +func connectWays(ways []*geo.PointSet, tryClockwiseWays bool) []*geo.PointSet { + for i, way := range ways { + for j, wayToCheck := range ways { + if i == j { + continue + } + + connectingWay := geo.NewPointSet() + + if way.Last().Equals(wayToCheck.First()) { + connectingWay = mergeWaysIntoWay([]*geo.PointSet{way, wayToCheck}) + } else if way.First().Equals(wayToCheck.Last()) { + connectingWay = mergeWaysIntoWay([]*geo.PointSet{wayToCheck, way}) + } else if tryClockwiseWays { + if way.Last().Equals(wayToCheck.Last()) { + reversedWay := reverseWay(wayToCheck) + + connectingWay = mergeWaysIntoWay([]*geo.PointSet{way, reversedWay}) + } else if way.First().Equals(wayToCheck.First()) { + reversedWay := reverseWay(way) + + connectingWay = mergeWaysIntoWay([]*geo.PointSet{reversedWay, wayToCheck}) + } + } + + if connectingWay.Length() > 0 { + ways = removeWaysByIndices([]int{i, j}, ways) + ways = append(ways, connectingWay) + + return ways + } + } + } + + return ways +} + +func mergeWaysIntoWay(ways []*geo.PointSet) *geo.PointSet { + singleWay := geo.NewPointSet() + + for _, way := range ways { + for i := 0; i < way.Length(); i++ { + singleWay.Push(way.GetAt(i)) + } + } + + return singleWay +} + +func removeWaysByIndices(indices []int, ways []*geo.PointSet) []*geo.PointSet { + waysWithoutIndices := make([]*geo.PointSet, 0) + + for wayIndex := 0; wayIndex < len(ways); wayIndex++ { + isMatchingAnyIndex := false + for _, index := range indices { + if index == wayIndex { + isMatchingAnyIndex = true + break + } + } + if !isMatchingAnyIndex { + waysWithoutIndices = append(waysWithoutIndices, ways[wayIndex]) + } + } + + return waysWithoutIndices +} + +func reverseWay(points *geo.PointSet) *geo.PointSet { + reversedWay := geo.NewPointSet() + + for i := points.Length() - 1; i >= 0; i-- { + reversedWay.Push(points.GetAt(i)) + } + + return reversedWay +} diff --git a/relation_centroid_test.go b/relation_centroid_test.go new file mode 100644 index 0000000..9b823dd --- /dev/null +++ b/relation_centroid_test.go @@ -0,0 +1,165 @@ +package main + +import ( + geo "github.com/paulmach/go.geo" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetPolygonFromWays(t *testing.T) { + var way1 = geo.NewPointSet() + way1.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + way1.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + + var way2 = geo.NewPointSet() + way2.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + way2.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + way2.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + + var way3 = geo.NewPointSet() + way3.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + way3.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + way3.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + way3.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + + var way4 = geo.NewPointSet() + way4.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + way4.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + + var toBeIgnoredWay = geo.NewPointSet() + toBeIgnoredWay.Push(geo.NewPoint(9.123456111111111, 53.891011111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.234567111111111, 53.910111111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.345678111111111, 53.101112111111111)) + + factoredPolygons := getPolygonsFromWays([]*geo.PointSet{way1, toBeIgnoredWay, way2, way3, way4}) + + expectedPolygon := geo.NewPointSet() + expectedPolygon.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + expectedPolygon.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + expectedPolygon.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + expectedPolygon.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + expectedPolygon.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + + assert.Len(t, factoredPolygons, 1) + assert.True(t, expectedPolygon.Equals(factoredPolygons[0])) +} + +func TestGetPolygonFromClockwiseWays(t *testing.T) { + var way1 = geo.NewPointSet() + way1.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + way1.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + + var way2 = geo.NewPointSet() + way2.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + way2.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + way2.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + + var way3 = geo.NewPointSet() + way3.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + way3.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + way3.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + way3.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + + var clockwiseWay = geo.NewPointSet() + clockwiseWay.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + clockwiseWay.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + + var toBeIgnoredWay = geo.NewPointSet() + toBeIgnoredWay.Push(geo.NewPoint(9.123456111111111, 53.891011111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.234567111111111, 53.910111111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.345678111111111, 53.101112111111111)) + + factoredPolygons := getPolygonsFromWays([]*geo.PointSet{way1, toBeIgnoredWay, way2, way3, clockwiseWay}) + expectedPolygon := geo.NewPointSet() + expectedPolygon.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + expectedPolygon.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + expectedPolygon.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + expectedPolygon.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + expectedPolygon.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + + assert.Len(t, factoredPolygons, 1) + assert.True(t, expectedPolygon.Equals(factoredPolygons[0])) +} + +func TestGetMultiplePolygonFromWays(t *testing.T) { + var way1 = geo.NewPointSet() + way1.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + way1.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + + var way2 = geo.NewPointSet() + way2.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + way2.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + way2.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + + var way3 = geo.NewPointSet() + way3.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + way3.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + way3.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + way3.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + + var way4 = geo.NewPointSet() + way4.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + way4.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + + var way5 = geo.NewPointSet() + way5.Push(geo.NewPoint(9.806362353265285, 53.55137991431488)) + way5.Push(geo.NewPoint(9.80620376765728, 53.55143369507137)) + + var way6 = geo.NewPointSet() + way6.Push(geo.NewPoint(9.80620376765728, 53.55143369507137)) + way6.Push(geo.NewPoint(9.806171916425228, 53.55138529239359)) + + var way7 = geo.NewPointSet() + way7.Push(geo.NewPoint(9.806171916425228, 53.55138529239359)) + way7.Push(geo.NewPoint(9.806314073503017, 53.551339479108485)) + + var way8 = geo.NewPointSet() + way8.Push(geo.NewPoint(9.806314073503017, 53.551339479108485)) + way8.Push(geo.NewPoint(9.806362353265285, 53.55137991431488)) + + var toBeIgnoredWay = geo.NewPointSet() + toBeIgnoredWay.Push(geo.NewPoint(9.123456111111111, 53.891011111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.234567111111111, 53.910111111111111)) + toBeIgnoredWay.Push(geo.NewPoint(9.345678111111111, 53.101112111111111)) + + factoredPolygons := getPolygonsFromWays([]*geo.PointSet{way1, toBeIgnoredWay, way2, way3, way4, way5, way6, way7, way8}) + + expectedPolygon1 := geo.NewPointSet() + expectedPolygon1.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + expectedPolygon1.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon1.Push(geo.NewPoint(9.869220256805418, 53.54410713060241)) + expectedPolygon1.Push(geo.NewPoint(9.80645354837179, 53.55131298705578)) + expectedPolygon1.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon1.Push(geo.NewPoint(9.806404933333397, 53.55129247064265)) + expectedPolygon1.Push(geo.NewPoint(9.80623260140419, 53.55131577569369)) + expectedPolygon1.Push(geo.NewPoint(9.806125648319721, 53.55134983689904)) + expectedPolygon1.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon1.Push(geo.NewPoint(9.806103184819221, 53.55139963393354)) + expectedPolygon1.Push(geo.NewPoint(9.869660139083862, 53.545229135979696)) + + expectedPolygon2 := geo.NewPointSet() + expectedPolygon2.Push(geo.NewPoint(9.806362353265285, 53.55137991431488)) + expectedPolygon2.Push(geo.NewPoint(9.80620376765728, 53.55143369507137)) + expectedPolygon2.Push(geo.NewPoint(9.80620376765728, 53.55143369507137)) + expectedPolygon2.Push(geo.NewPoint(9.806171916425228, 53.55138529239359)) + expectedPolygon2.Push(geo.NewPoint(9.806171916425228, 53.55138529239359)) + expectedPolygon2.Push(geo.NewPoint(9.806314073503017, 53.551339479108485)) + expectedPolygon2.Push(geo.NewPoint(9.806314073503017, 53.551339479108485)) + expectedPolygon2.Push(geo.NewPoint(9.806362353265285, 53.55137991431488)) + + assert.Len(t, factoredPolygons, 2) + assert.True(t, expectedPolygon1.Equals(factoredPolygons[0])) + assert.True(t, expectedPolygon2.Equals(factoredPolygons[1])) +} From 13b4990041e2cc170ddcdef7c77e0b4ebcb5e3ab Mon Sep 17 00:00:00 2001 From: Timon Masberg Date: Tue, 10 Aug 2021 00:57:57 +0200 Subject: [PATCH 7/7] feat(relation-outer-members-centroid): add relation centroid computing functionality to print and refactor entrance extraction --- centroid_test.go | 14 +++++----- pbf2json.go | 73 +++++++++++++++--------------------------------- 2 files changed, 29 insertions(+), 58 deletions(-) diff --git a/centroid_test.go b/centroid_test.go index 247f19d..f8da44a 100644 --- a/centroid_test.go +++ b/centroid_test.go @@ -12,7 +12,7 @@ func TestComputeCentroidWithEntranceNode(t *testing.T) { map[string]string{"lat": "1", "lon": "2", "entrance": "1"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "1", centroid["lat"]) assert.Equal(t, "2", centroid["lon"]) assert.Equal(t, +1.0, bounds.North()) @@ -29,7 +29,7 @@ func TestComputeCentroidWithMainEntranceNode(t *testing.T) { map[string]string{"lat": "-1", "lon": "-2", "entrance": "1", "wheelchair": "2"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "1", centroid["lat"]) assert.Equal(t, "2", centroid["lon"]) assert.Equal(t, +1.0, bounds.North()) @@ -45,7 +45,7 @@ func TestComputeCentroidWithAccessibleEntranceNode(t *testing.T) { map[string]string{"lat": "-1", "lon": "-2", "entrance": "1", "wheelchair": "2"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "-1", centroid["lat"]) assert.Equal(t, "-2", centroid["lon"]) assert.Equal(t, +0.0, bounds.North()) @@ -60,7 +60,7 @@ func TestComputeCentroidWithRegularEntranceNode(t *testing.T) { map[string]string{"lat": "0", "lon": "0", "entrance": "1"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "0", centroid["lat"]) assert.Equal(t, "0", centroid["lon"]) assert.Equal(t, +0.0, bounds.North()) @@ -79,7 +79,7 @@ func TestComputeCentroidForClosedPolygon(t *testing.T) { map[string]string{"lat": "1", "lon": "1"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "0.0000000", centroid["lat"]) assert.Equal(t, "0.0000000", centroid["lon"]) assert.Equal(t, +1.0, bounds.North()) @@ -100,7 +100,7 @@ func TestComputeCentroidForHillboroPublicLibrary(t *testing.T) { map[string]string{"lat": "45.5424694", "lon": "-122.9356798"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "45.5428760", centroid["lat"]) assert.Equal(t, "-122.9359955", centroid["lon"]) assert.Equal(t, +45.5433259, bounds.North()) @@ -117,7 +117,7 @@ func TestComputeCentroidForOpenLineString(t *testing.T) { map[string]string{"lat": "-1", "lon": "-1"}, } - var centroid, bounds = computeCentroidAndBounds(latlons) + var centroid, bounds = ComputeCentroidAndBounds(latlons) assert.Equal(t, "0.0000000", centroid["lat"]) assert.Equal(t, "0.0000000", centroid["lon"]) assert.Equal(t, +1.0, bounds.North()) diff --git a/pbf2json.go b/pbf2json.go index aa3ecf0..2ea5848 100644 --- a/pbf2json.go +++ b/pbf2json.go @@ -279,7 +279,7 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings } // compute centroid - centroid, bounds := computeCentroidAndBounds(latlons) + centroid, bounds := ComputeCentroidAndBounds(latlons) // trim tags v.Tags = trimTags(v.Tags) @@ -318,55 +318,15 @@ func print(d *osmpbf.Decoder, masks *BitmaskMap, db *leveldb.DB, config settings continue } - // best centroid and bounds to use - var largestArea = 0.0 - var centroid map[string]string - var bounds *geo.Bound + // consult https://wiki.openstreetmap.org/wiki/DE:Relation:multipolygon on how osm handles outer members + centroid, bounds := ComputeRelationCentroidAndBounds(memberWayLatLons) - // iterate over each way, selecting the largest way to use - // for the centroid and bbox - for _, latlons := range memberWayLatLons { - - // compute centroid - wayCentroid, wayBounds := computeCentroidAndBounds(latlons) - - // if for any reason we failed to find a valid bounds - if nil == wayBounds { - log.Println("[warn] failed to calculate bounds for relation member way") - continue - } - - area := GetAreaOfBounds(wayBounds) - - // find the way with the largest area - if area > largestArea { - largestArea = area - centroid = wayCentroid - bounds = wayBounds - } - } - - // if for any reason we failed to find a valid bounds - if nil == bounds { - log.Println("[warn] denormalize failed for relation:", v.ID, "no valid bounds") + if centroid == nil || bounds == nil { + // the relation is probably not a whole part of the osm dump + log.Printf("[warn] could not find centroid and bounds of %d", v.ID) continue } - // use 'admin_centre' node centroid where available - // note: only applies to 'boundary=administrative' relations - // see: https://github.com/pelias/pbf2json/pull/98 - if v.Tags["boundary"] == "administrative" { - for _, member := range v.Members { - if member.Type == 0 && member.Role == "admin_centre" { - if latlons, err := CacheLookupNodeByID(db, member.ID); err == nil { - latlons["type"] = "admin_centre" - centroid = latlons - break - } - } - } - } - // trim tags v.Tags = trimTags(v.Tags) @@ -682,10 +642,7 @@ func selectEntrance(entrances []map[string]string) map[string]string { return centroid } -// compute the centroid of a way and its bbox -func computeCentroidAndBounds(latlons []map[string]string) (map[string]string, *geo.Bound) { - - // check to see if there is a tagged entrance we can use. +func getEntrance(latlons []map[string]string) (bool, map[string]string, *geo.Bound) { var entrances []map[string]string for _, latlon := range latlons { if _, ok := latlon["entrance"]; ok { @@ -703,9 +660,23 @@ func computeCentroidAndBounds(latlons []map[string]string) (map[string]string, * // use the mapped entrance location where available if len(entrances) > 0 { - return selectEntrance(entrances), points.Bound() + return true, selectEntrance(entrances), points.Bound() } + return false, nil, nil +} + +// ComputeCentroidAndBounds compute the centroid of a way and its bbox for polygons and lines +func ComputeCentroidAndBounds(latlons []map[string]string) (map[string]string, *geo.Bound) { + hasEntrance, entranceLatLon, entranceBounds := getEntrance(latlons) + + if hasEntrance { + return entranceLatLon, entranceBounds + } + + // convert lat/lon map to geo.PointSet + points := LatLngMapToPointSet(latlons) + // determine if the way is a closed centroid or a linestring // by comparing first and last coordinates. isClosed := IsPointSetClosed(points)