forked from SomeoneElseOSM/mod_tile
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathopenstreetmap-tiles-update-rerender
executable file
·195 lines (158 loc) · 7.32 KB
/
openstreetmap-tiles-update-rerender
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
#!/bin/sh
set -e
#------------------------------------------------------------------------------
# AJT - change directory to mod_tile directory so that we can run replag
# and other things directly from this script when run from cron.
# Change the actual location to wherever installed locally.
#------------------------------------------------------------------------------
export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
cd /home/renderaccount/src/mod_tile/
#------------------------------------------------------------------------------
# The OSM2PGSQL_OPTIONS here only need setting if a tag transform script is
# in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and
# http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load
#------------------------------------------------------------------------------
OSMOSIS_BIN=osmosis
OSM2PGSQL_BIN=osm2pgsql
OSM2PGSQL_OPTIONS="-d gis"
#OSM2PGSQL_OPTIONS="--flat-nodes /path/to/flatnodes --hstore"
BASE_DIR=/var/lib/mod_tile
LOG_DIR=/var/log/tiles/
WORKOSM_DIR=$BASE_DIR/.osmosis
LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt
CHANGE_FILE=$BASE_DIR/changes.osc.gz
EXPIRY_FILE=$BASE_DIR/dirty_tiles
STOP_FILE=$BASE_DIR/stop.txt
OSMOSISLOG=$LOG_DIR/osmosis.log
PGSQLLOG=$LOG_DIR/osm2pgsql.log
EXPIRYLOG=$LOG_DIR/expiry.log
RUNLOG=$LOG_DIR/run.log
#------------------------------------------------------------------------------
# The tile expiry section below can re-render, delete or dirty expired tiles.
# By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered.
# "render_expired" can optionally delete (and/or dirty) tiles above a certail
# threshold rather than rendering them.
# Here we rerender tiles between zoom levels 13 and 18 and delete between 19
# and 20.
#------------------------------------------------------------------------------
EXPIRY_MINZOOM=13
EXPIRY_TOUCHFROM=21
EXPIRY_DELETEFROM=19
EXPIRY_MAXZOOM=20
#*************************************************************************
#*************************************************************************
m_info()
{
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
}
m_error()
{
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ [error] $1" >> "$RUNLOG"
m_info "resetting state"
/bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true
rm "$CHANGE_FILE" || true
rm "$EXPIRY_FILE.$$" || true
rm "$LOCK_FILE"
exit
}
m_ok()
{
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
}
getlock()
{
if [ -s $1 ]; then
if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then
return 1 #false
fi
fi
echo $$ >"$1"
return 0 #true
}
freelock()
{
rm "$1"
rm "$CHANGE_FILE"
}
if [ $# -eq 1 ] ; then
m_info "Initialising Osmosis replication system to $1"
mkdir $WORKOSM_DIR
$OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
else
# make sure the lockfile is removed when we exit and then claim it
if ! getlock "$LOCK_FILE"; then
m_info "pid `cat $LOCK_FILE` still running"
exit 3
fi
if [ -e $STOP_FILE ]; then
m_info "stopped"
exit 2
fi
# -----------------------------------------------------------------------------
# Add disk space check from https://github.com/zverik/regional
# -----------------------------------------------------------------------------
MIN_DISK_SPACE_MB=500
if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"`; then
m_info "there is less than $MIN_DISK_SPACE_MB MB left"
exit 4
fi
seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
m_ok "downloading diff"
if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
m_error "Osmosis error"
fi
# -----------------------------------------------------------------------------
# Add trim_osc.py call from https://github.com/zverik/regional
# The parameters passed correspond to the middle of England and Wales, and
# to the data that was originally loaded.
# -----------------------------------------------------------------------------
m_ok "filtering diff"
if ! /home/renderaccount/src/regional/trim_osc.py -d gis -b -14.17 48.85 2.12 61.27 -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>> "$RUNLOG"; then
m_error "Trim_osc error"
fi
m_ok "importing diff"
#------------------------------------------------------------------------------
# Previously openstreetmap-tiles-update-expire tried to dirty layer
# "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired
# tiles in range to the list (note the "-" rather than ":" in the "-e"
# parameter).
#------------------------------------------------------------------------------
if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
m_error "osm2pgsql error"
fi
#------------------------------------------------------------------------------
# The lockfile is normally removed before we expire tiles because that is
# something thatcan be done in parallel with further processing. In order to
# avoid rework, if actually rerendering is done rather than just deleting or
# dirtying, it makes sense to move it lower down.
#------------------------------------------------------------------------------
# m_ok "Import complete; removing lock file"
# freelock "$LOCK_FILE"
m_ok "expiring tiles"
#------------------------------------------------------------------------------
# When expiring tiles we need to define the style sheet if it's not "default".
# In this case it's "ajt".
# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and
# $EXPIRY_MAXZOOM were dirtied. We currently re-render
# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and
# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
# The default path to renderd.sock is fixed.
#
# Note no --touch-from=$EXPIRY_TOUCHFROM
#------------------------------------------------------------------------------
if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
m_info "Expiry failed"
fi
rm "$EXPIRY_FILE.$$"
#------------------------------------------------------------------------------
# Only remove the lock file after expiry (if system is slow we want to delay
# the next import, not have multiple render_expired processes running)
#------------------------------------------------------------------------------
freelock "$LOCK_FILE"
m_ok "Done with import"
fi