-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Download shape file dynamically from NDW, with if-none-match header.
- Loading branch information
Showing
12 changed files
with
268 additions
and
71 deletions.
There are no files selected for viewing
37 changes: 27 additions & 10 deletions
37
verkeersdrukte/src/main/java/nl/bertriksikken/verkeersdrukte/ndw/FileResponse.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,33 +1,50 @@ | ||
| package nl.bertriksikken.verkeersdrukte.ndw; | ||
|
|
||
| import com.google.common.collect.Iterables; | ||
| import jakarta.ws.rs.core.HttpHeaders; | ||
|
|
||
| import java.time.Instant; | ||
| import java.time.format.DateTimeFormatter; | ||
| import java.util.List; | ||
| import java.util.Locale; | ||
| import java.util.Map; | ||
|
|
||
| public final class FileResponse { | ||
|
|
||
| private final int code; | ||
| private final byte[] contents; | ||
| private final Instant lastModified; | ||
| private final Map<String, List<String>> headers; | ||
|
|
||
| FileResponse(byte[] contents, Instant lastModified) { | ||
| this.contents = contents; | ||
| this.lastModified = lastModified; | ||
| FileResponse(int code, Map<String, List<String>> headers, byte[] contents) { | ||
| this.code = code; | ||
| this.headers = headers; // do not copy, original map has special case-insensitive properties | ||
| this.contents = contents.clone(); | ||
| } | ||
|
|
||
| public static FileResponse create(byte[] contents, String lastModified) { | ||
| Instant date = DateTimeFormatter.RFC_1123_DATE_TIME.parse(lastModified, Instant::from); | ||
| return new FileResponse(contents, date); | ||
| public static FileResponse create(int code, Map<String, List<String>> headers, byte[] contents) { | ||
| return new FileResponse(code, headers, contents); | ||
| } | ||
|
|
||
| public static FileResponse empty() { | ||
| return new FileResponse(new byte[0], Instant.now()); | ||
| public int getCode() { | ||
| return code; | ||
| } | ||
|
|
||
| public byte[] getContents() { | ||
| return contents; | ||
| } | ||
|
|
||
| public Instant getLastModified() { | ||
| return lastModified; | ||
| String lastModified = Iterables.getFirst(headers.get(HttpHeaders.LAST_MODIFIED), ""); | ||
| return DateTimeFormatter.RFC_1123_DATE_TIME.parse(lastModified, Instant::from); | ||
| } | ||
|
|
||
| public String getEtag() { | ||
| List<String> values = headers.getOrDefault(HttpHeaders.ETAG, List.of()); | ||
| return Iterables.getFirst(values, ""); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return String.format(Locale.ROOT, "{code=%s,contents=%d bytes,headers=%s}", code, contents.length, headers); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
...eersdrukte/src/main/java/nl/bertriksikken/verkeersdrukte/traffic/ShapeFileDownloader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| package nl.bertriksikken.verkeersdrukte.traffic; | ||
|
|
||
| import nl.bertriksikken.geojson.FeatureCollection; | ||
| import nl.bertriksikken.shapefile.EShapeType; | ||
| import nl.bertriksikken.shapefile.ShapeFile; | ||
| import nl.bertriksikken.shapefile.ShapeRecord; | ||
| import nl.bertriksikken.verkeersdrukte.ndw.FileResponse; | ||
| import nl.bertriksikken.verkeersdrukte.ndw.NdwClient; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.ByteArrayInputStream; | ||
| import java.io.File; | ||
| import java.io.FileInputStream; | ||
| import java.io.FileOutputStream; | ||
| import java.io.IOException; | ||
| import java.util.zip.ZipEntry; | ||
| import java.util.zip.ZipInputStream; | ||
|
|
||
| public final class ShapeFileDownloader { | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(ShapeFileDownloader.class); | ||
|
|
||
| private final File folder; | ||
| private final NdwClient client; | ||
|
|
||
| private String etag = ""; | ||
| private ShapeFile shapeFile; | ||
|
|
||
| ShapeFileDownloader(File folder, NdwClient client) { | ||
| this.folder = folder; | ||
| this.client = client; | ||
| } | ||
|
|
||
| public boolean download() throws IOException { | ||
| FileResponse response = client.getShapeFile(etag); | ||
| if (response.getCode() != 200) { | ||
| LOG.info("Shapefile not downloaded, code {}", response.getCode()); | ||
| return false; | ||
| } | ||
|
|
||
| // remember etag for next time | ||
| etag = response.getEtag(); | ||
|
|
||
| // unzip | ||
| folder.mkdirs(); | ||
| deleteFiles(folder); | ||
| unzip(response.getContents(), folder); | ||
|
|
||
| // read shape file | ||
| try (FileInputStream shpStream = new FileInputStream(new File(folder, "Telpunten_WGS84.shp"))) { | ||
| try (FileInputStream dbfStream = new FileInputStream(new File(folder, "Telpunten_WGS84.dbf"))) { | ||
| shapeFile = ShapeFile.read(shpStream, dbfStream); | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| private void deleteFiles(File folder) { | ||
| for (File file : folder.listFiles(this::isTelpunt)) { | ||
| LOG.info("Deleting '{}'", file.getName()); | ||
| file.delete(); | ||
| } | ||
| } | ||
|
|
||
| private void unzip(byte[] contents, File folder) throws IOException { | ||
| ByteArrayInputStream bais = new ByteArrayInputStream(contents); | ||
| try (ZipInputStream zis = new ZipInputStream(bais)) { | ||
| ZipEntry entry; | ||
| while ((entry = zis.getNextEntry()) != null) { | ||
| String name = entry.getName(); | ||
| if (!entry.isDirectory() && isTelpunt(folder, name)) { | ||
| File file = new File(folder, name); | ||
| LOG.info("Unzipping {}", file.getName()); | ||
| unzipFile(zis, file); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void unzipFile(ZipInputStream zis, File outputFile) throws IOException { | ||
| try (FileOutputStream fos = new FileOutputStream(outputFile)) { | ||
| byte[] data = zis.readAllBytes(); | ||
| fos.write(data); | ||
| } | ||
| } | ||
|
|
||
| public FeatureCollection getFeatureCollection() throws IOException { | ||
| FeatureCollection collection = new FeatureCollection(); | ||
| for (ShapeRecord record : shapeFile.getRecords()) { | ||
| if (record.getType() == EShapeType.Point) { | ||
| ShapeRecord.Point point = (ShapeRecord.Point) record; | ||
| FeatureCollection.GeoJsonGeometry geometry = new FeatureCollection.PointGeometry(point.y, point.x); | ||
| FeatureCollection.Feature feature = new FeatureCollection.Feature(geometry); | ||
| record.getProperties().forEach(feature::addProperty); | ||
| collection.add(feature); | ||
| } | ||
| } | ||
| return collection; | ||
| } | ||
|
|
||
| private boolean isTelpunt(File dir, String name) { | ||
| return name.startsWith("Telpunten_WGS84"); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
verkeersdrukte/src/test/java/nl/bertriksikken/verkeersdrukte/ndw/FileResponseTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package nl.bertriksikken.verkeersdrukte.ndw; | ||
|
|
||
| import org.junit.Assert; | ||
| import org.junit.Test; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| public final class FileResponseTest { | ||
|
|
||
| @Test | ||
| public void test() { | ||
| byte[] contents = new byte[]{1, 2, 3}; | ||
| Map<String, List<String>> headers = Map.of("Last-Modified", List.of("Tue, 3 Jun 2008 11:05:30 GMT")); | ||
| FileResponse response = FileResponse.create(200, headers, contents); | ||
|
|
||
| Assert.assertArrayEquals(contents, response.getContents()); | ||
| Instant lastModified = response.getLastModified(); | ||
| Assert.assertNotNull(lastModified); | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.