diff --git a/homework/bill_gc.log b/homework/bill_gc.log new file mode 100644 index 0000000..e5e453e --- /dev/null +++ b/homework/bill_gc.log @@ -0,0 +1,16 @@ +Java HotSpot(TM) 64-Bit Server VM (25.20-b23) for windows-amd64 JRE (1.8.0_20-b26), built on Jul 30 2014 13:51:23 by "java_re" with MS VC++ 10.0 (VS2010) +Memory: 4k page, physical 8271460k(3665084k free), swap 10237540k(4273076k free) +CommandLine flags: -XX:InitialHeapSize=132343360 -XX:MaxHeapSize=2117493760 -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:-UseLargePagesIndividualAllocation -XX:+UseParallelGC +3.961: [GC (Allocation Failure) 23808K->21424K(125952K), 0.0080784 secs] +4.551: [GC (Allocation Failure) 53438K->52064K(125952K), 0.0110631 secs] +5.875: [GC (Allocation Failure) 83678K->82752K(125952K), 0.0129618 secs] +5.888: [Full GC (Ergonomics) 82752K->82716K(201216K), 0.0267797 secs] +6.612: [GC (Allocation Failure) 114034K->123740K(221184K), 0.0133295 secs] +6.625: [Full GC (Ergonomics) 123740K->113436K(273920K), 0.0038620 secs] +8.864: [GC (Allocation Failure) 165577K->205660K(282624K), 0.0266288 secs] +8.890: [Full GC (Ergonomics) 205660K->164636K(384512K), 0.0206266 secs] +10.402: [GC (Allocation Failure) 216917K->215868K(418304K), 0.0119434 secs] +14.423: [GC (Allocation Failure) 309699K->308060K(418304K), 0.0319572 secs] +14.455: [Full GC (Ergonomics) 308060K->307997K(593920K), 0.0288107 secs] +16.737: [GC (Allocation Failure) 401938K->410461K(655872K), 0.0324336 secs] +16.770: [Full GC (Ergonomics) 410461K->400157K(776192K), 0.0112225 secs] diff --git a/homework/bill_jvm32_gc.log b/homework/bill_jvm32_gc.log new file mode 100644 index 0000000..5590840 --- /dev/null +++ b/homework/bill_jvm32_gc.log @@ -0,0 +1,53 @@ +Java HotSpot(TM) Client VM (25.51-b03) for windows-x86 JRE (1.8.0_51-b16), built on Jun 8 2015 18:00:23 by "java_re" with MS VC++ 10.0 (VS2010) +Memory: 4k page, physical 8271460k(4177156k free), swap 10237540k(4761452k free) +CommandLine flags: -XX:ErrorFile=C:\Users\Bill\java_error_in_IDEA_%p.log -XX:InitialHeapSize=16777216 -XX:MaxHeapSize=1073741824 -XX:+PrintGC -XX:+PrintGCTimeStamps -XX:-UseLargePagesIndividualAllocation +0.322: [GC (Allocation Failure) 4416K->1040K(15872K), 0.0043299 secs] +0.398: [GC (Allocation Failure) 5456K->3128K(15872K), 0.0035010 secs] +0.480: [GC (Allocation Failure) 7544K->4146K(15872K), 0.0033180 secs] +0.508: [GC (Allocation Failure) 8562K->4319K(15872K), 0.0028429 secs] +0.515: [GC (Allocation Failure) 8726K->4362K(15872K), 0.0014477 secs] +0.546: [GC (Allocation Failure) 8778K->5109K(15872K), 0.0023394 secs] +0.571: [GC (Allocation Failure) 9525K->5040K(15872K), 0.0018788 secs] +0.619: [GC (Allocation Failure) 9456K->6239K(15872K), 0.0045714 secs] +0.639: [GC (Allocation Failure) 10655K->9123K(15872K), 0.0026407 secs] +0.653: [GC (Allocation Failure) 13539K->12218K(16640K), 0.0025906 secs] +0.656: [Full GC (Allocation Failure) 12218K->12218K(16640K), 0.0091288 secs] +0.694: [GC (Allocation Failure) 19579K->16967K(28348K), 0.0058126 secs] +0.722: [GC (Allocation Failure) 24834K->22086K(30052K), 0.0046649 secs] +0.727: [Full GC (Allocation Failure) 22086K->22086K(30052K), 0.0112068 secs] +0.778: [GC (Allocation Failure) 35490K->30043K(51368K), 0.0098116 secs] +0.830: [GC (Allocation Failure) 44315K->34207K(51368K), 0.0127499 secs] +0.882: [GC (Allocation Failure) 48479K->38497K(52776K), 0.0119572 secs] +0.894: [Full GC (Allocation Failure) 38497K->38497K(52776K), 0.0277656 secs] +1.057: [GC (Allocation Failure) 61409K->43770K(88932K), 0.0232897 secs] +1.567: [GC (Allocation Failure) 68410K->49161K(88932K), 0.0228224 secs] +1.774: [GC (Allocation Failure) 73801K->54209K(88932K), 0.0144856 secs] +2.660: [Full GC (Metadata GC Threshold) 69102K->42564K(88932K), 0.0449064 secs] +3.355: [GC (Allocation Failure) 71044K->47328K(102944K), 0.0108748 secs] +3.904: [GC (Allocation Failure) 75808K->52938K(102944K), 0.0210788 secs] +4.027: [GC (Allocation Failure) 81418K->56317K(102944K), 0.0125392 secs] +4.355: [GC (Allocation Failure) 84797K->57277K(102944K), 0.0092810 secs] +4.707: [GC (Allocation Failure) 85757K->60982K(102944K), 0.0120037 secs] +5.090: [Full GC (Metadata GC Threshold) 75873K->58456K(102944K), 0.0899523 secs] +6.323: [GC (Allocation Failure) 97496K->67513K(141332K), 0.0300465 secs] +7.357: [GC (Allocation Failure) 102999K->78400K(141332K), 0.0350410 secs] +7.672: [Full GC (Metadata GC Threshold) 87567K->65335K(141332K), 0.1162865 secs] +9.394: [GC (Allocation Failure) 108983K->70979K(157984K), 0.0274511 secs] +10.300: [GC (Allocation Failure) 114627K->74486K(157984K), 0.0302167 secs] +10.983: [GC (Allocation Failure) 118134K->76286K(157984K), 0.0271287 secs] +11.780: [GC (Allocation Failure) 119934K->79470K(157984K), 0.0233156 secs] +13.111: [GC (Allocation Failure) 123118K->81341K(157984K), 0.0249988 secs] +13.268: [Full GC (Metadata GC Threshold) 85218K->66155K(157984K), 0.2353434 secs] +14.551: [GC (Allocation Failure) 110379K->69711K(159988K), 0.0204552 secs] +15.486: [GC (Allocation Failure) 113935K->71958K(159988K), 0.0192550 secs] +16.629: [GC (Allocation Failure) 116182K->75935K(159988K), 0.0278797 secs] +17.686: [GC (Allocation Failure) 120159K->83147K(159988K), 0.0294754 secs] +18.268: [GC (Allocation Failure) 127371K->94111K(159988K), 0.0380868 secs] +18.725: [GC (Allocation Failure) 138335K->105327K(159988K), 0.0443552 secs] +23.391: [GC (Allocation Failure) 149551K->113632K(159988K), 0.0503248 secs] +57.356: [GC (Allocation Failure) 157856K->118215K(163324K), 0.0372537 secs] +57.394: [Full GC (Allocation Failure) 118215K->78406K(163324K), 0.2369982 secs] +92.345: [GC (Allocation Failure) 130821K->81545K(189624K), 0.0168981 secs] +117.588: [GC (Allocation Failure) 133961K->82092K(189624K), 0.0214815 secs] +139.777: [GC (Allocation Failure) 134508K->82945K(189624K), 0.0225465 secs] +219.740: [GC (Allocation Failure) 135361K->82582K(189624K), 0.0140721 secs] diff --git a/homework/heap_dump_compare.png b/homework/heap_dump_compare.png new file mode 100644 index 0000000..5d43d19 Binary files /dev/null and b/homework/heap_dump_compare.png differ diff --git a/homework/jvmgc_2.png b/homework/jvmgc_2.png new file mode 100644 index 0000000..aa759c4 Binary files /dev/null and b/homework/jvmgc_2.png differ diff --git a/homework/jvmgc_3.png b/homework/jvmgc_3.png new file mode 100644 index 0000000..38820b9 Binary files /dev/null and b/homework/jvmgc_3.png differ diff --git a/homework/jvmgc_4.png b/homework/jvmgc_4.png new file mode 100644 index 0000000..06017d8 Binary files /dev/null and b/homework/jvmgc_4.png differ diff --git a/homework/settings.xml b/homework/settings.xml new file mode 100644 index 0000000..70af453 --- /dev/null +++ b/homework/settings.xml @@ -0,0 +1,12 @@ + + + + CN + OSChina Central + http://maven.oschina.net/content/groups/public/ + central + + + /home/bill/.m2/repository + diff --git a/pom.xml b/pom.xml index 904b2df..decdf8d 100644 --- a/pom.xml +++ b/pom.xml @@ -17,18 +17,37 @@ 1.8 + + org.springframework.boot + spring-boot-maven-plugin + + + + org.springframework.boot + spring-boot-starter-parent + 1.3.5.RELEASE + + + + UTF-8 + 1.8 + + - com.fasterxml.jackson.core - jackson-annotations - 2.4.5 + org.springframework.boot + spring-boot-starter-web - com.fasterxml.jackson.core - jackson-databind - 2.4.5 + org.springframework.boot + spring-boot-starter-data-jpa + + + org.springframework.boot + spring-boot-starter-test + test org.apache.commons @@ -36,22 +55,35 @@ 1.2 - junit - junit - 4.12 + org.dbunit + dbunit + 2.5.1 test + + + junit + junit + + - com.google.guava - guava - 13.0.1 + com.github.springtestdbunit + spring-test-dbunit + 1.2.1 + test com.google.guava guava 13.0.1 - + + com.h2database + h2 + runtime + + + - \ No newline at end of file + diff --git a/src/main/java/com/epam/data/EnrichRoadAccident.java b/src/main/java/com/epam/data/EnrichRoadAccident.java new file mode 100644 index 0000000..0aaba9f --- /dev/null +++ b/src/main/java/com/epam/data/EnrichRoadAccident.java @@ -0,0 +1,37 @@ +package com.epam.data; + +import java.time.LocalTime; + +/** + * Created by bill on 16-5-2. + */ +public class EnrichRoadAccident extends RoadAccident { + private String ForceContact; + private TimeOfDay timeOfDay; + + public EnrichRoadAccident(RoadAccidentBuilder builder) { + super(builder); + } + +/* + public EnrichRoadAccident(RoadAccident roadAccident) { + super(builder); + } +*/ + + public String getForceContact() { + return ForceContact; + } + + public void setForceContact(String forceContact) { + ForceContact = forceContact; + } + + public TimeOfDay getTimeOfDay() { + return timeOfDay; + } + + public void setTimeOfDay(LocalTime time) { + this.timeOfDay = TimeOfDay.getTimeOfDay(time); + } +} diff --git a/src/main/java/com/epam/data/RoadAccident.java b/src/main/java/com/epam/data/RoadAccident.java index 9548fa5..6441ed2 100644 --- a/src/main/java/com/epam/data/RoadAccident.java +++ b/src/main/java/com/epam/data/RoadAccident.java @@ -146,4 +146,35 @@ public String getRoadSurfaceConditions() { public void setRoadSurfaceConditions(String roadSurfaceConditions) { this.roadSurfaceConditions = roadSurfaceConditions; } + + // Patch for home work 3, should extend RoadAccident class + private String ForceContact = null; + private TimeOfDay timeOfDay; + + public String getForceContact() { + return ForceContact; + } + + public void setForceContact(String forceContact) { + ForceContact = forceContact; + } + + public TimeOfDay getTimeOfDay() { + return timeOfDay; + } + + public void setTimeOfDay(LocalTime time) { + this.timeOfDay = TimeOfDay.getTimeOfDay(time); + } + + public String toCSV() { + return String.join(",", + accidentId, + policeForce, + ForceContact, + time.toString(), + timeOfDay.toString() + ); + } + } diff --git a/src/main/java/com/epam/data/TimeOfDay.java b/src/main/java/com/epam/data/TimeOfDay.java new file mode 100644 index 0000000..4b0558f --- /dev/null +++ b/src/main/java/com/epam/data/TimeOfDay.java @@ -0,0 +1,50 @@ +package com.epam.data; + +import java.time.LocalTime; + +/** + * Created by bill on 16-5-2. + * + */ + +public enum TimeOfDay { + MORNING, AFTERNOON, EVENING, NIGHT; + + public static TimeOfDay getTimeOfDay(LocalTime time) { + int hour = time.getHour(); + switch (hour/6) { + case 0: + return TimeOfDay.NIGHT; + case 1: + return TimeOfDay.MORNING; + case 2: + return TimeOfDay.AFTERNOON; + case 3: + default: + return TimeOfDay.EVENING; + } + } + + public int getCategory() { + if(this.equals(TimeOfDay.MORNING) | this.equals(TimeOfDay.AFTERNOON)) { + return 1; + } else { + return 0; + } + } +} + +/* + MORNING - 6 am to 12 pm + AFTERNOON - 12 pm to 6 pm + EVENING - 6 pm to 12 am + NIGHT - 12 am to 6 am + MORNING, + AFTERNOON(AFTERNOON"), + EVENING("EVENING"), + NIGHT("NIGHT"); + MORNING, + AFTERNOON, + EVENING, + NIGHT; +*/ diff --git a/src/main/java/com/epam/dataservice/AccidentBatchLoader.java b/src/main/java/com/epam/dataservice/AccidentBatchLoader.java index 016a2b1..b39854c 100644 --- a/src/main/java/com/epam/dataservice/AccidentBatchLoader.java +++ b/src/main/java/com/epam/dataservice/AccidentBatchLoader.java @@ -1,6 +1,7 @@ package com.epam.dataservice; +import com.epam.data.EnrichRoadAccident; import com.epam.data.RoadAccident; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -13,9 +14,12 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; public class AccidentBatchLoader implements Callable { + private static AtomicInteger serialNo = new AtomicInteger(); + private String taskName = this.getClass().getSimpleName() + serialNo.incrementAndGet(); private Integer batchSize; private BlockingQueue> dataQueue; private String dataFileName; @@ -42,11 +46,13 @@ public Integer call() throws Exception { dataCount = dataCount + roadAccidentBatch.size(); if(roadAccidentBatch.isEmpty()){ isDataLoadFinished = true; + System.out.println(taskName + " Finished"); }else{ ++batchCount; System.out.println(" Completed reading " + dataCount + " in " + batchCount + " batches for " + dataFileName); +// System.out.println(taskName + " Trying to put, Queue.remainingCapacity()="+dataQueue.remainingCapacity()); + dataQueue.put(roadAccidentBatch); } - dataQueue.put(roadAccidentBatch); } //dataQueue.put(roadAccidentBatch); //Epmty batch can be used as identifier for end of record production return dataCount; @@ -61,7 +67,7 @@ private List getNextBatch(Iterator recordIterator){ List roadAccidentBatch = new ArrayList(); int recordCount = 0; RoadAccident roadAccidentItem = null; - while(recordCount <= batchSize && recordIterator.hasNext() ){ + while(recordCount < batchSize && recordIterator.hasNext() ){ roadAccidentItem = roadAccidentParser.parseRecord(recordIterator.next()); if(roadAccidentItem != null){ roadAccidentBatch.add(roadAccidentItem); diff --git a/src/main/java/com/epam/dataservice/AccidentBatchProcessor.java b/src/main/java/com/epam/dataservice/AccidentBatchProcessor.java new file mode 100644 index 0000000..c29ed5c --- /dev/null +++ b/src/main/java/com/epam/dataservice/AccidentBatchProcessor.java @@ -0,0 +1,97 @@ +package com.epam.dataservice; + +import com.epam.data.RoadAccident; + +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by bill on 16-5-1. + */ +public class AccidentBatchProcessor implements Runnable { + private static AtomicInteger serialNo = new AtomicInteger(); +// For record Enrichment + private static PoliceForceService policeForceService = new PoliceForceService(); + private static ExecutorService enrichExecutor = Executors.newFixedThreadPool(HomeWork3.MAX_ENRICH_THREADS); + private CompletionService enrichCompletionService = new ExecutorCompletionService(enrichExecutor); + + private String taskName = getClass().getSimpleName() + serialNo.incrementAndGet(); + private BlockingQueue> readQueue; + private List> writeQueues; + + public AccidentBatchProcessor(BlockingQueue> readQueue, List> writeQueues){ + this.readQueue = readQueue; + this.writeQueues = writeQueues; + } + + @Override + public void run() { + int dataCounter = 0; + List consumedData = null; + Boolean done = false; + try { + while (!done) { + consumedData = readQueue.take(); + if (consumedData != null ) { + if(!consumedData.isEmpty()) { + dataCounter += consumedData.size(); + System.out.println(" Consumed " + dataCounter + " records from " + taskName); + enrichRoadAccident(consumedData); + } else { + done = true; + System.out.println(taskName + " Get empty data," + " Quit"); + } + } else { + System.out.println(taskName + " Abort"); + break; + } + } +// System.out.println(taskName + " Finished"); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + public void enrichRoadAccident(List consumedData) { + for (RoadAccident record : consumedData) { + enrichCompletionService.submit(new enrichRecord(record)); + } + for (int i = 0; i < consumedData.size(); i++) { + try { + RoadAccident record = enrichCompletionService.take().get(); + int index = record.getTimeOfDay().getCategory(); + writeQueues.get(index).put(record); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + } + + private class enrichRecord implements Callable { + private RoadAccident record; + public enrichRecord(RoadAccident record) { + this.record = record; + } + + @Override + public RoadAccident call() { + String contactNo = null; + try { + contactNo = policeForceService.getContactNo(record.getPoliceForce()); + } catch (RuntimeException e) { + System.out.println("Catch policeForceService.getContactNo exception"); + e.printStackTrace(); + contactNo = null; + } + record.setForceContact(contactNo); + record.setTimeOfDay(record.getTime()); +// System.out.println(taskName + " policeForceService=" + contactNo); + return record; + } + } + +} diff --git a/src/main/java/com/epam/dataservice/AccidentBatchWriter.java b/src/main/java/com/epam/dataservice/AccidentBatchWriter.java new file mode 100644 index 0000000..74ca964 --- /dev/null +++ b/src/main/java/com/epam/dataservice/AccidentBatchWriter.java @@ -0,0 +1,50 @@ +package com.epam.dataservice; + +import com.epam.data.RoadAccident; + +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by bill on 16-5-2. + */ +public class AccidentBatchWriter implements Runnable { + private static AtomicInteger serialNo = new AtomicInteger(); + private String taskName = this.getClass().getSimpleName() + serialNo.incrementAndGet(); + private BlockingQueue dataQueue; + private String dataFileName; + + public AccidentBatchWriter(BlockingQueue dataQueue, String dataFileName) { + this.dataQueue = dataQueue; + this.dataFileName = "src/main/resources/" + dataFileName; + } + + @Override + public void run() { + Boolean done = false; + int dataCounter = 0; + + System.out.println(taskName + " Write to file: " + dataFileName); + try (PrintWriter csv = new PrintWriter(dataFileName)) { + // CSV Header + while (!done) { + RoadAccident record = dataQueue.take(); + if(record.getAccidentId() != null) { + // write to file + csv.println(record.toCSV()); + dataCounter++; + } else { + done = true; + System.out.println(taskName + " Quit, Total write: " + dataCounter); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + } +} diff --git a/src/main/java/com/epam/dataservice/HomeWork3.java b/src/main/java/com/epam/dataservice/HomeWork3.java new file mode 100644 index 0000000..9c89b43 --- /dev/null +++ b/src/main/java/com/epam/dataservice/HomeWork3.java @@ -0,0 +1,154 @@ +package com.epam.dataservice; + +import com.epam.data.RoadAccident; +import com.epam.data.RoadAccidentBuilder; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Created by bill on 16-5-1. + */ +public class HomeWork3 { + public final int MAX_READ_THREADS = 2; + public final int MAX_PROC_THREADS = 4; + public static final int MAX_ENRICH_THREADS = 100; + public final int MAX_WRITE_THREADS = 2; + public static final int MAX_BATCH_SIZE = 40000; + public static final String outputFileNames[] = {"NighttimeAccidents.csv", "DaytimeAccidents.csv"}; + + private LinkedList fileList; + private BlockingQueue> readQueue; + private ExecutorService readExecutor; + private List> readTaskList = new ArrayList<>(); + + private ExecutorService procExecutor; + + private List> writeQueues = new ArrayList<>(MAX_WRITE_THREADS); + private ExecutorService writeExecutor; + + public HomeWork3(LinkedList fileList) { + this.fileList = fileList; + } + + public void startTask() { + readQueue = new ArrayBlockingQueue>(MAX_PROC_THREADS); + for (int i = 0; i < MAX_WRITE_THREADS; i++) { + writeQueues.add( new ArrayBlockingQueue(MAX_BATCH_SIZE/2) ); + } + + readExecutor = Executors.newFixedThreadPool(MAX_READ_THREADS); + for (String file : fileList ) { + FutureTask readTask = new FutureTask( + new AccidentBatchLoader(MAX_BATCH_SIZE,readQueue,file)); + readTaskList.add(readTask); + readExecutor.submit(readTask); + } + + procExecutor = Executors.newFixedThreadPool(MAX_PROC_THREADS); + for (int i = 0; i < MAX_PROC_THREADS; i++) { + procExecutor.submit(new AccidentBatchProcessor(readQueue, writeQueues)); + } + + writeExecutor = Executors.newFixedThreadPool(MAX_WRITE_THREADS); + for (int i = 0; i < MAX_WRITE_THREADS; i++) { + writeExecutor.submit(new AccidentBatchWriter(writeQueues.get(i),outputFileNames[i])); + } + System.out.println("All task submitted"); + } + + public void run() { + startTask(); + + try { + readQueueFinish(); + endDatLoading(procExecutor); + for (int i = 0; i < MAX_WRITE_THREADS; i++) { + System.out.println("Send quit message to writeQueue, count: " + Integer.toString(i+1)); + writeQueues.get(i).put( new RoadAccidentBuilder(null).build() ); + } + endDatLoading(writeExecutor); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void readQueueFinish() throws Exception { + Boolean taskStatus; + while(!readTaskList.isEmpty()) { + Iterator> iter = readTaskList.iterator(); + while (iter.hasNext()) { + if ( iter.next().isDone() ) { + iter.remove(); + } + } + TimeUnit.SECONDS.sleep(1); + } + + readExecutor.submit(new Runnable() { + @Override + public void run() { + List emptyAccidentList = new ArrayList(1); + for (int i = 0; i < MAX_PROC_THREADS; i++) { + System.out.println("Send quit message to readQueue, count: " + Integer.toString(i+1)); + try { + readQueue.put(emptyAccidentList); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + }); + + readExecutor.shutdown(); + try { + taskStatus = readExecutor.awaitTermination(5, TimeUnit.MINUTES); + if (taskStatus) { + System.out.println("Data loading finished"); + } else { + System.out.println("Data loading task timeout"); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + public static void main(String[] args) throws Exception{ + } + + private static class ReadTerminator implements Runnable { + public static List emptyAccidentList = new ArrayList(1); + private BlockingQueue> dataQueue; + + public ReadTerminator(BlockingQueue> dataQueue) { + this.dataQueue = dataQueue; + } + + @Override + public void run() { + try { + System.out.println("Send quit message"); + dataQueue.put(emptyAccidentList); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + private static void endDatLoading(ExecutorService executor){ + executor.shutdown(); + try { + if (executor.awaitTermination(10, TimeUnit.SECONDS)) { + System.out.println("All task finished for " + executor.getClass().getSimpleName()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } +// System.out.println("Data loading finished"); + } + +} diff --git a/src/main/java/com/epam/dataservice/PoliceForceService.java b/src/main/java/com/epam/dataservice/PoliceForceService.java index d871bd4..d6b581f 100644 --- a/src/main/java/com/epam/dataservice/PoliceForceService.java +++ b/src/main/java/com/epam/dataservice/PoliceForceService.java @@ -18,8 +18,8 @@ public class PoliceForceService{ private final String PHONE_PREFIX = "13163862"; private Map forceMap = new HashMap<>(); private AtomicInteger executionCount = new AtomicInteger(0); - private final int HALT_AT_EXECUTION = 10; - private final long HALT_FOR = 1000 * 2; + private final int HALT_AT_EXECUTION = 20000; + private final long HALT_FOR = 100 * 2; public PoliceForceService(){ init(); @@ -41,15 +41,16 @@ public void init(){ } public String getContactNo(String policeForceName){ - if(executionCount.get() == HALT_AT_EXECUTION){ + if(executionCount.incrementAndGet() % HALT_AT_EXECUTION == 0){ try { + System.out.println("Oops:"+executionCount.get()); Thread.sleep(HALT_FOR); } catch (InterruptedException e) { throw new RuntimeException(e); } } Integer idx = forceMap.get(policeForceName); - executionCount.incrementAndGet(); +// executionCount.incrementAndGet(); return idx == null ? PHONE_PREFIX : PHONE_PREFIX + idx.toString() ; } diff --git a/src/main/java/com/epam/dataservice/RoadAccidentParser.java b/src/main/java/com/epam/dataservice/RoadAccidentParser.java index c78509c..39dfa8f 100644 --- a/src/main/java/com/epam/dataservice/RoadAccidentParser.java +++ b/src/main/java/com/epam/dataservice/RoadAccidentParser.java @@ -1,5 +1,6 @@ package com.epam.dataservice; +import com.epam.data.EnrichRoadAccident; import com.epam.data.RoadAccident; import com.epam.data.RoadAccidentBuilder; import org.apache.commons.csv.CSVRecord; diff --git a/src/main/java/com/epam/processor/DataProcessor.java b/src/main/java/com/epam/processor/DataProcessor.java index d2f37d7..e52aca5 100644 --- a/src/main/java/com/epam/processor/DataProcessor.java +++ b/src/main/java/com/epam/processor/DataProcessor.java @@ -1,11 +1,12 @@ package com.epam.processor; import com.epam.data.RoadAccident; -import com.google.common.collect.Multimap; +import com.google.common.base.*; +import com.google.common.collect.*; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.Optional; +import java.util.stream.Collectors; /** * This is to be completed by mentees @@ -18,7 +19,12 @@ public DataProcessor(List roadAccidentList){ this.roadAccidentList = roadAccidentList; } - + public static boolean inRect(RoadAccident roadAccident, float minLongitude, float maxLongitude, float minLatitude, float maxLatitude) { + float longitude = roadAccident.getLongitude(); + float latitude = roadAccident.getLatitude(); + return ( (longitude>=minLongitude && longitude<=maxLongitude) + && (latitude>=minLatitude && latitude<=maxLatitude) ); + } // First try to solve task using java 7 style for processing collections /** @@ -27,6 +33,12 @@ public DataProcessor(List roadAccidentList){ * @return */ public RoadAccident getAccidentByIndex7(String index){ + Iterator iter = roadAccidentList.iterator(); + while(iter.hasNext()) { + RoadAccident rec = iter.next(); + if (rec.getAccidentId().equals(index)) + return rec; + } return null; } @@ -40,7 +52,13 @@ public RoadAccident getAccidentByIndex7(String index){ * @return */ public Collection getAccidentsByLocation7(float minLongitude, float maxLongitude, float minLatitude, float maxLatitude){ - return null; + List result = new ArrayList<>(); + for (RoadAccident rec : roadAccidentList ) { + if ( inRect(rec, minLongitude, maxLongitude, minLatitude, maxLatitude) ) { + result.add(rec); + } + } + return result; } /** @@ -51,7 +69,16 @@ public Collection getAccidentsByLocation7(float minLongitude, floa * @return */ public Map getCountByRoadSurfaceCondition7(){ - return null; + Map result = new HashMap<>(); + Multiset roadSurfaceCondition = HashMultiset.create(); + for (RoadAccident rec : roadAccidentList ) { + roadSurfaceCondition.add(rec.getRoadSurfaceConditions()); + } + for (String key : roadSurfaceCondition.elementSet() ) { +// System.out.println(key+" count："+roadSurfaceCondition.count(key)); + result.put(key, Long.valueOf(roadSurfaceCondition.count(key))); + } + return result; } /** @@ -59,8 +86,43 @@ public Map getCountByRoadSurfaceCondition7(){ * as example if there were 10 accidence in rain, 5 in snow, 6 in sunny and 1 in foggy, then your result list should contain {rain, sunny, snow} - top three in decreasing order * @return */ + public List getTopThreeWeatherCondition6(){ + + Multiset weatherConditions = HashMultiset.create(); + for (RoadAccident rec : roadAccidentList ) { + weatherConditions.add(rec.getWeatherConditions()); + } + ImmutableSortedMap.Builder builder = new ImmutableSortedMap + .Builder(Ordering.natural().reverse()); + for (String key : weatherConditions.elementSet() ) { + String countStr = String.format("%08d %s", weatherConditions.count(key), key); + System.out.println(countStr); + builder.put(countStr, key); + } + ImmutableSortedMap sortedConditions = builder.build(); + List result = sortedConditions.values().asList(); +// System.out.println(result); +/* + // If no duplicate count value for WeatherCondition, we can use BiMap + // Another Solution + Multimap sortedConditions = TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural()); + for (String key : weatherConditions.elementSet() ) { + sortedConditions.put(weatherConditions.count(key), key); + } + List result = ImmutableList.copyOf(sortedConditions.values()); +*/ + return result.subList(0,3); + } + public List getTopThreeWeatherCondition7(){ - return null; + Multiset weatherConditions = HashMultiset.create(); + for (RoadAccident rec : roadAccidentList ) { + weatherConditions.add(rec.getWeatherConditions()); + } + List> topKList = Ordering + .from(Comparator.>comparingInt(e -> e.getCount())) + .greatestOf(weatherConditions.entrySet(),3); + return Lists.transform(topKList, e -> e.getElement()); } /** @@ -71,7 +133,11 @@ public List getTopThreeWeatherCondition7(){ * @return */ public Multimap getAccidentIdsGroupedByAuthority7(){ - return null; + ImmutableMultimap.Builder builder = new ImmutableMultimap.Builder(); + for (RoadAccident rec : roadAccidentList ) { + builder.put(rec.getDistrictAuthority(), rec.getAccidentId()); + } + return builder.build(); } @@ -80,7 +146,10 @@ public Multimap getAccidentIdsGroupedByAuthority7(){ public RoadAccident getAccidentByIndex(String index){ - return null; + return roadAccidentList.stream() + .filter(r -> r.getAccidentId().equals(index)) + .findFirst().orElse(null); + } @@ -93,7 +162,9 @@ public RoadAccident getAccidentByIndex(String index){ * @return */ public Collection getAccidentsByLocation(float minLongitude, float maxLongitude, float minLatitude, float maxLatitude){ - return null; + return roadAccidentList.stream() + .filter(r -> inRect(r, minLongitude, maxLongitude, minLatitude, maxLatitude) ) + .collect(Collectors.toList()); } /** @@ -101,7 +172,14 @@ public Collection getAccidentsByLocation(float minLongitude, float * @return */ public List getTopThreeWeatherCondition(){ - return null; + Map weatherCondition = roadAccidentList.stream() + .collect(Collectors.groupingBy(RoadAccident::getWeatherConditions, + Collectors.counting())); + return weatherCondition.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(3) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); } /** @@ -109,7 +187,11 @@ public List getTopThreeWeatherCondition(){ * @return */ public Map getCountByRoadSurfaceCondition(){ - return null; + Map roadSurfaceCondition = roadAccidentList.stream() + .collect(Collectors.groupingBy(RoadAccident::getRoadSurfaceConditions, + Collectors.counting())); + + return roadSurfaceCondition; } /** @@ -117,7 +199,11 @@ public Map getCountByRoadSurfaceCondition(){ * @return */ public Map> getAccidentIdsGroupedByAuthority(){ - return null; + Map> authority = roadAccidentList.stream() + .collect(Collectors.groupingBy(RoadAccident::getDistrictAuthority, + Collectors.mapping(RoadAccident::getAccidentId, Collectors.toList()))); + + return authority; } } diff --git a/src/main/java/com/epam/springboot/AccidentsRestApplication.java b/src/main/java/com/epam/springboot/AccidentsRestApplication.java new file mode 100644 index 0000000..307dc51 --- /dev/null +++ b/src/main/java/com/epam/springboot/AccidentsRestApplication.java @@ -0,0 +1,45 @@ +package com.epam.springboot; + +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.modal.RoadConditions; +import com.epam.springboot.repository.AccidentRepository; +import com.epam.springboot.repository.AccidentService; +import com.epam.springboot.repository.AccidentServiceImpl; +import com.epam.springboot.repository.RoadConditionRepository; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ApplicationContext; + +import java.util.List; + +/** + * Created by bill on 16-5-22. + */ + +@SpringBootApplication +public class AccidentsRestApplication { + public static ApplicationContext getContext() { + return context; + } + + private static ApplicationContext context; + public static void main(String[] args) { + SpringApplication springApplication = new SpringApplication(); + context = SpringApplication.run(AccidentsRestApplication.class, args); + initDB(context); + } + + private static void initDB(ApplicationContext ctx){ +// AccidentRepository accidentRepository= ctx.getBean(AccidentRepository.class); +// RoadConditionRepository roadConditionRepository= context.getBean(RoadConditionRepository.class); +// roadConditionRepository.save(new RoadConditions(12,"Test Sample")); +// AccidentServiceImpl accidentService = (AccidentServiceImpl)context.getBean("accidentService"); +// AccidentServiceImpl accidentServiceImpl = context.getBean(AccidentServiceImpl.class); +// accidentRepository.save(new Accidents("200901BS70001")); +// accidentRepository.save(new Accidents("200901BS70002",3)); +// List accidentsList = accidentRepository.findAll(); +// System.out.println(accidentsList); + } +// @Bean + +} diff --git a/src/main/java/com/epam/springboot/controller/AccidentController.java b/src/main/java/com/epam/springboot/controller/AccidentController.java new file mode 100644 index 0000000..b1f3efa --- /dev/null +++ b/src/main/java/com/epam/springboot/controller/AccidentController.java @@ -0,0 +1,73 @@ +package com.epam.springboot.controller; + +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.repository.AccidentRepository; +import com.epam.springboot.repository.AccidentService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.orm.ObjectRetrievalFailureException; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +/** + * Created by bill on 16-5-22. + */ +@RestController +//@RequestMapping(value = "/api/") +public class AccidentController { + private static org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(AccidentController.class); + @Autowired + AccidentRepository repository; + @Autowired + AccidentService accidentService; + +/* + @ExceptionHandler + void handleIllegalArgumentException(IllegalArgumentException e, + HttpServletResponse response) throws IOException { + response.sendError(HttpStatus.BAD_REQUEST.value()); + } +*/ + + @RequestMapping(method= RequestMethod.GET, value= "/accidents", headers="Accept=application/json") + public @ResponseBody + List accidents() { + return repository.findAll(); + } + + @RequestMapping(method= RequestMethod.GET, value= "/accidents/{id}", headers="Accept=application/json") + public @ResponseBody + Accidents accidents(@PathVariable String id){ + Accidents accidents= repository.findOne(id); + return accidents; + } + + @RequestMapping(method= RequestMethod.POST, value= "/accidents", headers="Accept=application/json") + public @ResponseBody String save(@RequestBody Accidents accidents){ + logger.info("creating Accidents: "+ accidents.toString()); + accidentService.createAccident(accidents); + return accidents.getId(); + } + + @RequestMapping(method= RequestMethod.PUT, value= "/accidents/{id}", headers="Accept=application/json") + public @ResponseBody String update(@RequestBody Accidents accidents, @PathVariable String id){ + logger.info("Update Accidents with: " + accidents); + if(!accidents.getId().equals(id)) { + logger.error("Expect " + id + " , but get " + accidents.getId()); + throw new IllegalArgumentException("Expect " + id + " , but get " + accidents.getId()); + } + if (repository.findOne(id) == null) { + throw new ObjectRetrievalFailureException(repository.getClass(), accidents); + } + repository.save(accidents); + return accidents.getId(); + } + + @RequestMapping(method= RequestMethod.DELETE, value= "/accidents/{id}", headers="Accept=application/json") + public @ResponseBody void delete(@PathVariable String id){ + logger.info("Delete Accidents with ID=" + id); + logger.info("[Before] Count=" + repository.count()); + repository.delete(id); + logger.info("[After] Count=" + repository.count()); + } +} diff --git a/src/main/java/com/epam/springboot/controller/HelloController.java b/src/main/java/com/epam/springboot/controller/HelloController.java new file mode 100644 index 0000000..eb9a4e0 --- /dev/null +++ b/src/main/java/com/epam/springboot/controller/HelloController.java @@ -0,0 +1,17 @@ +package com.epam.springboot.controller; + +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Created by bill on 16-5-22. + */ +@RestController +public class HelloController { + + @RequestMapping(value = "hello/{name}") + public String hello(@PathVariable String name){ + return "hello, "+ name; + } +} diff --git a/src/main/java/com/epam/springboot/controller/RestControllerAdvice.java b/src/main/java/com/epam/springboot/controller/RestControllerAdvice.java new file mode 100644 index 0000000..3d925ea --- /dev/null +++ b/src/main/java/com/epam/springboot/controller/RestControllerAdvice.java @@ -0,0 +1,42 @@ +package com.epam.springboot.controller; + +import org.springframework.http.HttpStatus; +import org.springframework.orm.ObjectRetrievalFailureException; +import org.springframework.web.bind.annotation.*; + +/** + * Created by bill on 16-6-5. + */ +@ControllerAdvice(annotations = RestController.class) +public class RestControllerAdvice { + @ExceptionHandler(ObjectRetrievalFailureException.class) + @ResponseStatus(HttpStatus.NOT_FOUND) + @ResponseBody + public String handle(ObjectRetrievalFailureException e) { + return e.getMessage() + "\n"; + } + + @ExceptionHandler(IllegalArgumentException.class) + @ResponseStatus(HttpStatus.BAD_REQUEST) + @ResponseBody + public String handle(IllegalArgumentException e) { + return e.getMessage() + "\n"; + } + + @ExceptionHandler(value=RuntimeException.class) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + @ResponseBody + public String handle(RuntimeException e) { + return "INTERNAL_SERVER_ERROR_RUNTIME"; + } + + @ExceptionHandler(value=Exception.class) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + @ResponseBody + public String handle(Exception e) { + //log it + return "INTERNAL_SERVER_ERROR_ALL_EXCEPTION"; + + } + +} diff --git a/src/main/java/com/epam/springboot/controller/WeatherController.java b/src/main/java/com/epam/springboot/controller/WeatherController.java new file mode 100644 index 0000000..d843eb3 --- /dev/null +++ b/src/main/java/com/epam/springboot/controller/WeatherController.java @@ -0,0 +1,94 @@ +package com.epam.springboot.controller; + +import com.epam.springboot.modal.WeatherConditions; +import com.epam.springboot.repository.WeatherConditionRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.List; + +/** + * Created by bill on 16-5-29. + */ +@RestController +@RequestMapping(value = "/weather") +public class WeatherController { + private static org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(WeatherController.class); + @Autowired + private WeatherConditionRepository weatherConditionRepository; + + @ExceptionHandler + void handleIllegalArgumentException(IllegalArgumentException e, + HttpServletResponse response) throws IOException { + response.sendError(HttpStatus.BAD_REQUEST.value()); + } + + @RequestMapping(method= RequestMethod.GET, value= "/init", headers="Accept=application/json") + public @ResponseBody String init() { + weatherConditionRepository.saveAndFlush(new WeatherConditions(1,"First")); + weatherConditionRepository.saveAndFlush(new WeatherConditions(2,"Second")); + weatherConditionRepository.saveAndFlush(new WeatherConditions(3,"Third")); + return "Init 2 records"; + } + + @RequestMapping(method= RequestMethod.GET, value= "/test", headers="Accept=application/json") + public @ResponseBody + WeatherConditions testWeatherConditions() { + return new WeatherConditions(20,"Test20"); + } + + @RequestMapping(method= RequestMethod.GET, headers="Accept=application/json") + public @ResponseBody + List getWeatherConditions() { + List result = weatherConditionRepository.findAll(); + logger.info(result); + return result; + } + + @RequestMapping(method= RequestMethod.GET, value= "/{id}", headers="Accept=application/json") + public @ResponseBody + WeatherConditions getWeatherCondition(@PathVariable Integer id){ + WeatherConditions weatherConditions = new WeatherConditions(20,"Test20"); + logger.warn("Autowired weatherConditionRepository is: " + weatherConditionRepository); + if (weatherConditionRepository != null) { + weatherConditions = weatherConditionRepository.findOne(id); + } + logger.info(weatherConditions); + return weatherConditions; + } + + @RequestMapping(method= RequestMethod.POST, headers="Accept=application/json") + public @ResponseBody + WeatherConditions create (@RequestBody WeatherConditions weatherConditions) { + logger.info("Create " + weatherConditions); + return weatherConditionRepository.save(weatherConditions); + } + + @RequestMapping(method= RequestMethod.PUT, value= "/{id}", headers="Accept=application/json") + public @ResponseBody + WeatherConditions put (@PathVariable Integer id, @RequestBody WeatherConditions weatherConditions) { + WeatherConditions target = weatherConditionRepository.findOne(id); + if ((target != null) && (weatherConditions.getCode().equals(target.getCode()))) { + logger.info("Put: Update " + target + " With " + weatherConditions); + target.setLabel(weatherConditions.getLabel()); + return weatherConditionRepository.save(target); + } else { + logger.error("Try to update id=" + id + " With " + weatherConditions ); + throw new IllegalArgumentException(); +// return null; + } + } + + @RequestMapping(method= RequestMethod.DELETE, value= "/{id}", headers="Accept=application/json") + public @ResponseBody + WeatherConditions delete (@PathVariable Integer id) { + WeatherConditions weatherConditions = weatherConditionRepository.findOne(id); + weatherConditionRepository.delete(id); + logger.info("Delete " + weatherConditions); + return weatherConditions; + } + +} diff --git a/src/main/java/com/epam/springboot/modal/Accidents.java b/src/main/java/com/epam/springboot/modal/Accidents.java new file mode 100644 index 0000000..94a989e --- /dev/null +++ b/src/main/java/com/epam/springboot/modal/Accidents.java @@ -0,0 +1,224 @@ +package com.epam.springboot.modal; + +import javax.persistence.*; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +/** + * Created by bill on 16-5-22. + */ +@Entity +@Table(name="Accidents") +public class Accidents { + + @Id + @Column(name="Accident_Index") + private String id; + + @ManyToOne(cascade = CascadeType.PERSIST,fetch = FetchType.EAGER) + @JoinColumn(name = "Weather_Conditions") // Column name in table Accidents + private WeatherConditions weatherConditions; + + @ManyToOne(cascade = CascadeType.PERSIST,fetch = FetchType.EAGER) + @JoinColumn(name = "Road_Surface_Conditions") // Column name in table Accidents + private RoadConditions roadSurfaceConditions; + + @Column(name="Date") + @Temporal(TemporalType.DATE) + private java.util.Date date; + + @Column(name="Time") + private String time; + + private double longitude = 0.0; + private double latitude = 0.0; + @Column(name="Police_Force") + private Integer policeForce = 1; + @Column(name="Accident_Severity") + private Integer severity = 1; + @Column(name="Number_of_Vehicles") + private Integer numberOfVehicles = 0; + @Column(name="Number_of_Casualties") + private Integer numberOfCasualties = 0; + @Column(name="Day_of_Week") + private Integer dayOfWeek; + @Column(name="Local_Authority") + private Integer districtAuthority = 1; + @Column(name="Light_Conditions") + private Integer lightConditions = -1; + + protected Accidents() { + } + + public Accidents(String id) { + this.id = id; + } + + public Accidents(String id, WeatherConditions weatherConditions, RoadConditions roadSurfaceConditions, Date date) { + this.id = id; + this.weatherConditions = weatherConditions; + this.roadSurfaceConditions = roadSurfaceConditions; + this.date = date; + } + + public Accidents(String id, WeatherConditions weatherConditions, RoadConditions roadSurfaceConditions, Date date, String time, double longitude, double latitude, Integer policeForce, Integer severity, Integer numberOfVehicles, Integer numberOfCasualties, Integer dayOfWeek, Integer districtAuthority, Integer lightConditions) { + this.id = id; + this.weatherConditions = weatherConditions; + this.roadSurfaceConditions = roadSurfaceConditions; + this.date = date; + this.time = time; + this.longitude = longitude; + this.latitude = latitude; + this.policeForce = policeForce; + this.severity = severity; + this.numberOfVehicles = numberOfVehicles; + this.numberOfCasualties = numberOfCasualties; + this.dayOfWeek = dayOfWeek; + this.districtAuthority = districtAuthority; + this.lightConditions = lightConditions; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public void setWeatherConditions(WeatherConditions weatherConditions) { + this.weatherConditions = weatherConditions; + } + public WeatherConditions getWeatherConditions() { + return weatherConditions; + } + + public void setRoadSurfaceConditions(RoadConditions roadSurfaceConditions) { + this.roadSurfaceConditions = roadSurfaceConditions; + } + + public RoadConditions getRoadSurfaceConditions() { + return roadSurfaceConditions; + } + + public Date getDate() { + return date; + } + + public void setDate(Date date) { + this.date = date; + } + + public String getTime() { + return time; + } + + public void setTime(String time) { + this.time = time; + } + + public double getLongitude() { + return longitude; + } + + public void setLongitude(double longitude) { + this.longitude = longitude; + } + + public double getLatitude() { + return latitude; + } + + public void setLatitude(double latitude) { + this.latitude = latitude; + } + + public Integer getPoliceForce() { + return policeForce; + } + + public void setPoliceForce(Integer policeForce) { + this.policeForce = policeForce; + } + + public Integer getSeverity() { + return severity; + } + + public void setSeverity(Integer severity) { + this.severity = severity; + } + + public Integer getNumberOfVehicles() { + return numberOfVehicles; + } + + public void setNumberOfVehicles(Integer numberOfVehicles) { + this.numberOfVehicles = numberOfVehicles; + } + + public Integer getNumberOfCasualties() { + return numberOfCasualties; + } + + public void setNumberOfCasualties(Integer numberOfCasualties) { + this.numberOfCasualties = numberOfCasualties; + } + + public Integer getDayOfWeek() { + return dayOfWeek; + } + + public void setDayOfWeek(Integer dayOfWeek) { + this.dayOfWeek = dayOfWeek; + } + + public Integer getDistrictAuthority() { + return districtAuthority; + } + + public void setDistrictAuthority(Integer districtAuthority) { + this.districtAuthority = districtAuthority; + } + + public Integer getLightConditions() { + return lightConditions; + } + + public void setLightConditions(Integer lightConditions) { + this.lightConditions = lightConditions; + } + + @Transient + // use getLocalTime will cause DBUnit fail + public LocalTime convertLocalTime() { + LocalTime localTime = LocalTime.parse(time, DateTimeFormatter.ofPattern("H:mm")); + return localTime; + } + + + @Override + public String toString() { + return "Accidents{" + + "id='" + id + '\'' + + ", longitude=" + longitude + + ", latitude=" + latitude + + ", policeForce=" + policeForce + + ", severity=" + severity + + ", numberOfVehicles=" + numberOfVehicles + + ", numberOfCasualties=" + numberOfCasualties + + ", date=" + date + + ", dayOfWeek=" + dayOfWeek + + ", time=" + time + + ", districtAuthority=" + districtAuthority + + ", lightConditions=" + lightConditions + + ", weatherConditions=" + weatherConditions + + ", roadSurfaceConditions=" + roadSurfaceConditions + + '}' + "\n"; + } + +} diff --git a/src/main/java/com/epam/springboot/modal/RoadConditions.java b/src/main/java/com/epam/springboot/modal/RoadConditions.java new file mode 100644 index 0000000..23a4731 --- /dev/null +++ b/src/main/java/com/epam/springboot/modal/RoadConditions.java @@ -0,0 +1,60 @@ +package com.epam.springboot.modal; + +import com.epam.springboot.modal.Accidents; +import javax.persistence.*; +import java.io.Serializable; +import java.util.Set; + +/** + * Created by Bill on 2016/5/25. + */ +@Entity +@Table(name="road_surface") +public class RoadConditions implements Serializable { + private static final long serialVersionUID = 1L; + + @Id + @Column(name="code") + private Integer code; + @Column(name="label") + private String label; + +// @OneToMany(mappedBy = "roadSurfaceConditions", cascade = CascadeType.REMOVE, fetch = FetchType.LAZY) +// private Set accidents; + + protected RoadConditions() {} + + public RoadConditions(Integer code) { + this.code = code; + this.label = "Invalid"; + } + + public RoadConditions(Integer code, String label) { + this.code = code; + this.label = label; + } + + public Integer getCode() { + return code; + } + + public void setCode(Integer code) { + this.code = code; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + @Override + public String toString() { + return "RoadConditions{" + + "code=" + code + + ", label='" + label + '\'' + + '}'; + } +} diff --git a/src/main/java/com/epam/springboot/modal/WeatherConditions.java b/src/main/java/com/epam/springboot/modal/WeatherConditions.java new file mode 100644 index 0000000..4568e95 --- /dev/null +++ b/src/main/java/com/epam/springboot/modal/WeatherConditions.java @@ -0,0 +1,52 @@ +package com.epam.springboot.modal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +/** + * Created by bill on 16-5-28. + */ +@Entity +@Table(name="weather_conditions") +public class WeatherConditions { + + @Id + @Column(name="code") + private Integer code; + @Column(name="label") + private String label; + + protected WeatherConditions() { + } + + public WeatherConditions(Integer code, String label) { + this.code = code; + this.label = label; + } + + public WeatherConditions(Integer code) { + this.code = code; + } + + public Integer getCode() { + return code; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + @Override + public String toString() { + return "WeatherConditions{" + + "code=" + code + + ", label='" + label + '\'' + + '}'; + } +} diff --git a/src/main/java/com/epam/springboot/repository/AccidentRepository.java b/src/main/java/com/epam/springboot/repository/AccidentRepository.java new file mode 100644 index 0000000..ec934a6 --- /dev/null +++ b/src/main/java/com/epam/springboot/repository/AccidentRepository.java @@ -0,0 +1,39 @@ +package com.epam.springboot.repository; + +import com.epam.data.RoadAccident; +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.modal.RoadConditions; +import com.epam.springboot.modal.WeatherConditions; +import org.springframework.cglib.core.Local; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.time.LocalDate; +import java.util.Date; +import java.util.List; + +/** + * Created by bill on 16-5-22. + */ +@Repository +public interface AccidentRepository extends JpaRepository { + + List findByRoadSurfaceConditions(RoadConditions roadCondition); + + Integer countByRoadSurfaceConditions(RoadConditions roadCondition); + + List findByDate(Date date); + List findByDateBetween(Date date1, Date date2); + + List findByRoadSurfaceConditionsAndDateBetween(RoadConditions roadCondition, Date date1, Date date2); + Integer countByWeatherConditionsAndDateBetween(WeatherConditions weatherConditions, Date date1, Date date2); + + //(clearAutomatically = true) + @Modifying + @Query("UPDATE Accidents r SET r.time = :timeOfDay WHERE r.id = :accidentId") // Use java entity name + int updateTime(@Param("accidentId") String id, @Param("timeOfDay") String timeOfDay); + +} diff --git a/src/main/java/com/epam/springboot/repository/AccidentService.java b/src/main/java/com/epam/springboot/repository/AccidentService.java new file mode 100644 index 0000000..d6f9108 --- /dev/null +++ b/src/main/java/com/epam/springboot/repository/AccidentService.java @@ -0,0 +1,16 @@ +package com.epam.springboot.repository; + +import com.epam.springboot.modal.Accidents; + +import javax.persistence.criteria.CriteriaBuilder; +import java.util.Map; + +/** + * Created by bill on 16-5-28. + */ +public interface AccidentService { + Map getAccidentCountGroupByRoadCondition(); + Map getAccidentCountGroupByWeatherConditionAndYear(String year); + int updateAccidentTimeByDate(String strDate); + Accidents createAccident(Accidents accidents); +} diff --git a/src/main/java/com/epam/springboot/repository/AccidentServiceImpl.java b/src/main/java/com/epam/springboot/repository/AccidentServiceImpl.java new file mode 100644 index 0000000..3f1b97f --- /dev/null +++ b/src/main/java/com/epam/springboot/repository/AccidentServiceImpl.java @@ -0,0 +1,100 @@ +package com.epam.springboot.repository; + +import com.epam.data.TimeOfDay; +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.modal.RoadConditions; +import com.epam.springboot.modal.WeatherConditions; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalTime; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by bill on 16-5-28. + */ +@Component("accidentService") +@Transactional +public class AccidentServiceImpl implements AccidentService { + + public static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + + private final AccidentRepository repository; + private final RoadConditionRepository roadConditionRepository; + private final WeatherConditionRepository weatherConditionRepository; + + @Autowired + public AccidentServiceImpl(AccidentRepository repository, + RoadConditionRepository roadConditionRepository, + WeatherConditionRepository weatherConditionRepository) { + this.repository = repository; + this.roadConditionRepository = roadConditionRepository; + this.weatherConditionRepository = weatherConditionRepository; + } + + @Override + public Map getAccidentCountGroupByRoadCondition() { + Map result = new HashMap<>(); + List roadConditionsList = roadConditionRepository.findAll(); + + for (RoadConditions roadCondition : roadConditionsList ) { + result.put(roadCondition.getLabel(), + repository.countByRoadSurfaceConditions(roadCondition)); + } + return result; + } + + @Override + public Map getAccidentCountGroupByWeatherConditionAndYear(String year) { + Map result = new HashMap<>(); + try { + Date date1 = dateFormat.parse(year + "-01-01"); + Date date2 = dateFormat.parse(year + "-12-31"); + List weatherConditionsList = weatherConditionRepository.findAll(); + + for (WeatherConditions weatherCondition : weatherConditionsList ) { + result.put(weatherCondition.getLabel(), + repository.countByWeatherConditionsAndDateBetween(weatherCondition, date1, date2)); + } + } catch (ParseException e) { + e.printStackTrace(); + } + + return result; + } + + @Override + public int updateAccidentTimeByDate(String strDate) { + try { + Date date = dateFormat.parse(strDate); + List accidentsList = repository.findByDate(date); + for (Accidents accidents : accidentsList) { + String id = accidents.getId(); + LocalTime localTime = accidents.convertLocalTime(); + String timeOfDay = TimeOfDay.getTimeOfDay(localTime).toString(); +// System.out.println(id + " => Covert " + localTime + " to " + timeOfDay); + repository.updateTime(id,timeOfDay); + } + return accidentsList.size(); + } catch (ParseException e) { + e.printStackTrace(); + } + return 0; + } + + @Override + public Accidents createAccident(Accidents accidents) { + Integer id; + id = accidents.getWeatherConditions().getCode(); + accidents.setWeatherConditions(weatherConditionRepository.findOne(id)); + id = accidents.getRoadSurfaceConditions().getCode(); + accidents.setRoadSurfaceConditions(roadConditionRepository.findOne(id)); + return repository.save(accidents); + } +} diff --git a/src/main/java/com/epam/springboot/repository/RoadConditionRepository.java b/src/main/java/com/epam/springboot/repository/RoadConditionRepository.java new file mode 100644 index 0000000..171bc8f --- /dev/null +++ b/src/main/java/com/epam/springboot/repository/RoadConditionRepository.java @@ -0,0 +1,12 @@ +package com.epam.springboot.repository; + +import com.epam.springboot.modal.RoadConditions; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by Bill on 2016/5/26. + */ +@Repository +public interface RoadConditionRepository extends JpaRepository { +} diff --git a/src/main/java/com/epam/springboot/repository/WeatherConditionRepository.java b/src/main/java/com/epam/springboot/repository/WeatherConditionRepository.java new file mode 100644 index 0000000..eeeafff --- /dev/null +++ b/src/main/java/com/epam/springboot/repository/WeatherConditionRepository.java @@ -0,0 +1,12 @@ +package com.epam.springboot.repository; + +import com.epam.springboot.modal.WeatherConditions; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * Created by bill on 16-5-28. + */ +@Repository +public interface WeatherConditionRepository extends JpaRepository { +} diff --git a/src/main/resources/Accidents.csv b/src/main/resources/Accidents.csv new file mode 100644 index 0000000..ec3aa3b --- /dev/null +++ b/src/main/resources/Accidents.csv @@ -0,0 +1,17 @@ +Accident_Index,Longitude,Latitude,Police_Force,Accident_Severity,Number_of_Vehicles,Number_of_Casualties,Date,Day_of_Week,Time,Local_Authority,Light_Conditions,Weather_Conditions,Road_Surface_Conditions +200901BS70001,-0.201349,51.512273,1,2,2,1,2009-01-01,5,15:11,12,1,1,1 +200901BS70002,-0.199248,51.514399,1,2,2,11,2009-01-05,2,10:59,12,1,1,2 +200901BS70003,-0.179599,51.486668,1,3,2,1,2009-01-04,1,14:19,12,1,1,1 +200901BS70004,-0.20311,51.507804,1,2,2,1,2009-01-05,2,8:10,12,1,8,4 +200901BS70005,-0.173445,51.482076,1,2,2,1,2009-01-06,3,17:25,12,4,1,1 +200901BS70006,-0.185525,51.493415,1,3,2,3,2009-01-01,5,11:48,12,1,1,1 +200901BS70007,-0.178561,51.480177,1,2,2,1,2009-01-08,5,13:58,12,1,1,1 +200901BS70008,-0.178524,51.491957,1,3,1,1,2009-01-02,6,13:18,12,1,1,1 +200901BS70009,-0.167395,51.49646,1,3,1,2,2009-01-07,4,12:15,12,1,1,1 +200901BS70010,-0.183275,51.48115,1,3,1,1,2009-01-10,7,9:52,12,1,8,2 +200901BS70011,-0.173445,51.482076,1,3,2,1,2009-01-07,4,0:09,12,4,1,1 +200901BS70012,-0.183013,51.494995,1,3,1,1,2009-01-16,6,17:49,12,4,1,1 +200901BS70015,-0.206779,51.498778,1,3,2,1,2009-01-12,2,14:00,12,1,2,2 +200901BS70016,-0.209082,51.506187,1,3,2,1,2009-01-09,6,8:15,12,1,2,2 +200901BS70017,-0.169548,51.493077,1,3,2,1,2009-01-17,7,12:15,12,1,1,1 +200901BS70019,-0.173445,51.482076,1,2,2,1,2009-01-25,1,22:05,12,4,1,1 diff --git a/src/main/resources/prep_accidents.awk b/src/main/resources/prep_accidents.awk new file mode 100755 index 0000000..4fe6972 --- /dev/null +++ b/src/main/resources/prep_accidents.awk @@ -0,0 +1,18 @@ +#!/usr/bin/awk -f + +# Process original Road Accidents CSV file for testing + +BEGIN { + FS=","; + OFS=","; +# format = "%s,%s,%06.0s,%s,%s\n"; +} +{ + if (NR > 1) { + split($8,array,"/"); +# printf("%s-%s-%s\n", array[3],array[2],array[1]) + $8 = sprintf("%s-%s-%s", array[3],array[2],array[1]); + } + print($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); +} + diff --git a/src/main/resources/table-ordering-all.txt b/src/main/resources/table-ordering-all.txt new file mode 100644 index 0000000..b49c461 --- /dev/null +++ b/src/main/resources/table-ordering-all.txt @@ -0,0 +1,6 @@ +accident_severity +disrict_authority +light_conditions +police_force +road_surface +weather_conditions \ No newline at end of file diff --git a/src/main/resources/table-ordering.txt b/src/main/resources/table-ordering.txt new file mode 100644 index 0000000..4ebc4be --- /dev/null +++ b/src/main/resources/table-ordering.txt @@ -0,0 +1,3 @@ +weather_conditions +road_surface +Accidents diff --git a/src/test/java/com/epam/dataservice/HomeWork3Test.java b/src/test/java/com/epam/dataservice/HomeWork3Test.java new file mode 100644 index 0000000..4f45fc4 --- /dev/null +++ b/src/test/java/com/epam/dataservice/HomeWork3Test.java @@ -0,0 +1,157 @@ +package com.epam.dataservice; + +import com.epam.data.AccidentsDataLoader; +import com.epam.data.RoadAccident; +import com.epam.data.RoadAccidentBuilder; +import com.epam.data.TimeOfDay; +import com.epam.processor.DataProcessor; +import org.hamcrest.core.AnyOf; +import org.hamcrest.core.StringStartsWith; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.*; + +import static org.hamcrest.core.Is.is; +import org.hamcrest.core.IsInstanceOf; +import org.hamcrest.core.IsNull; +import static org.hamcrest.core.IsCollectionContaining.hasItems; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsInstanceOf.*; +import static org.junit.Assert.*; + + +/** + * Created by bill on 16-5-1. + */ +public class HomeWork3Test { + public static LinkedList inputFiles = new LinkedList<>(); + private static HomeWork3 homeWork3; + + @BeforeClass + public static void loadData(){ + for (int i = 2009; i <= 2012 ; i++) { + inputFiles.add("src/main/resources/DfTRoadSafety_Accidents_" + Integer.toString(i) +".csv"); + } + homeWork3 = new HomeWork3(inputFiles); + } + + private RoadAccident createRoadAccident(String id, String time) { + RoadAccident roadAccident; + if (id == null) { + roadAccident = new RoadAccidentBuilder(null).build(); + } else { + roadAccident = new RoadAccidentBuilder(id) + .withTime(LocalTime.parse(time, DateTimeFormatter.ISO_LOCAL_TIME)) + .build(); + } + return roadAccident; + } + + @Test + public void testAccidentBatchProcessor() throws Exception { + String[] times = {"00:00:00", "06:00:00", "17:59:59", "23:59:59"}; + BlockingQueue> readQueue = new ArrayBlockingQueue>(2); + List> writeQueues = new ArrayList<>(2); + for (int i = 0; i < 2; i++) { + writeQueues.add( new ArrayBlockingQueue(2) ); + } + ExecutorService procExecutor = Executors.newFixedThreadPool(2); + for (int i = 0; i < 2; i++) { + procExecutor.submit(new AccidentBatchProcessor(readQueue, writeQueues)); + } + for (String time : times ) { + List roadAccidentList = new ArrayList(); + roadAccidentList.add(createRoadAccident(time, time)); + readQueue.put(roadAccidentList); + } +// TimeUnit.SECONDS.sleep(1); + + RoadAccident roadAccident; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + BlockingQueue writeQueue = writeQueues.get(j); + System.out.println("Q" + j + " Size=" + writeQueue.size()); + roadAccident = writeQueue.take(); + System.out.println(roadAccident.toCSV()); + assertThat(roadAccident, instanceOf(RoadAccident.class)); +/* + if (j==0) { + assertThat(roadAccident.getTimeOfDay(), isIn(Arrays.asList(TimeOfDay.EVENING, TimeOfDay.NIGHT)); + } else { + assertThat(roadAccident.getTimeOfDay(), isIn(Arrays.asList(TimeOfDay.MORNING, TimeOfDay.AFTERNOON)); + } +*/ + assertThat(roadAccident.getTimeOfDay().getCategory(), equalTo(j)); + assertThat(roadAccident.getForceContact(), StringStartsWith.startsWith("13163862")); + } +// TimeUnit.SECONDS.sleep(1); + } + for (int i = 0; i < 2; i++) { + assertThat(writeQueues.get(i).isEmpty(), equalTo(true)); + } + assertThat(readQueue.isEmpty(), equalTo(true)); + procExecutor.shutdown(); + } + + @Test + public void testMain() { + homeWork3.run(); + } + + @Test + public void testEnum() { + TimeOfDay timeOfDay; + timeOfDay = TimeOfDay.getTimeOfDay(LocalTime.parse("09:56:00", DateTimeFormatter.ISO_LOCAL_TIME)); + assertThat(timeOfDay, equalTo(TimeOfDay.MORNING)); + assertThat(timeOfDay.getCategory(), equalTo(1)); + timeOfDay = TimeOfDay.getTimeOfDay(LocalTime.parse("03:56:00", DateTimeFormatter.ISO_LOCAL_TIME)); + assertThat(timeOfDay, equalTo(TimeOfDay.NIGHT)); + assertThat(timeOfDay.getCategory(), equalTo(0)); + } + + @Test + public void testPoliceForceService() { + PoliceForceService policeForceService = new PoliceForceService(); + assertThat(policeForceService.getContactNo("Norfolk"), equalTo("1316386236")); + assertThat(policeForceService.getContactNo("London"), equalTo("13163862")); + assertThat(policeForceService.getContactNo(""), equalTo("13163862")); + assertThat(policeForceService.getContactNo(null), equalTo("13163862")); +// assertThat(policeForceService.getContactNo("London"), IsNull.nullValue()); + } + + @Test + public void sqlDateTimeTest() throws ParseException { + SimpleDateFormat inputFormat = new SimpleDateFormat("dd/MM/yyyy"); + SimpleDateFormat outputFormat = new SimpleDateFormat("yyyy-MM-dd"); + String csvDate = "28/05/2016"; + Date date = inputFormat.parse(csvDate); + System.out.println("Date is: " + date + " year=" + outputFormat.format(date)); +// LocalDate localDate = new LocalDate(date); + SimpleDateFormat shortFormat = new SimpleDateFormat("yyyy"); + System.out.println("short date: " + shortFormat.parse("2016")); + + } +} +/* + Read accident files for different years and then we need to populate following 2 fields for each record. + 1. ForceContact -- Use PoliceForceService.getContactNo(String forceName) to get contact no. + Assume you dont have control on this method and this method can be time consuming or may even block randomly. + Assume this can be a I/O service for example some web service + + 2. TimeosDay -- Add 1 field DayTime which can have 4 different values as below based on accident time. + MORNING - 6 am to 12 pm + AFTERNOON - 12 pm to 6 pm + EVENING - 6 pm to 12 am + NIGHT - 12 am to 6 am + + And then we need to generate 2 files + 1. File which contains all day time accdent data for the years (MORNING, AFTERNOON) -- DaytimeAccidents.csv + 2. File which contains all night time accdent data for the years (EVENING, NIGHT) -- NighttimeAccidents.csv +*/ diff --git a/src/test/java/com/epam/dataservice/HomeWork5Test.java b/src/test/java/com/epam/dataservice/HomeWork5Test.java new file mode 100644 index 0000000..0dce4ce --- /dev/null +++ b/src/test/java/com/epam/dataservice/HomeWork5Test.java @@ -0,0 +1,184 @@ +package com.epam.dataservice; + +import com.epam.springboot.AccidentsRestApplication; +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.modal.RoadConditions; +import com.epam.springboot.repository.AccidentRepository; +import com.epam.springboot.repository.AccidentService; +import com.epam.springboot.repository.AccidentServiceImpl; +import com.epam.springboot.repository.RoadConditionRepository; +import com.github.springtestdbunit.DbUnitTestExecutionListener; +import com.github.springtestdbunit.annotation.DatabaseSetup; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.IntegrationTest; +import org.springframework.boot.test.SpringApplicationConfiguration; +import org.springframework.boot.test.TestRestTemplate; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.web.client.RestTemplate; + +import java.net.URL; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; + +/** + * Created by bill on 16-5-22. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringApplicationConfiguration(classes = AccidentsRestApplication.class) //MockServletContext +@TestExecutionListeners({DependencyInjectionTestExecutionListener.class, +// DirtiesContextTestExecutionListener.class, +// TransactionalTestExecutionListener.class, + DbUnitTestExecutionListener.class}) +@WebAppConfiguration +@IntegrationTest("server.port:0") +@DatabaseSetup("/sampleData.xml") +//@DatabaseTearDown(type = DatabaseOperation.DELETE_ALL, value = { ItemRepositoryIT.DATASET }) +public class HomeWork5Test { + @Autowired + RoadConditionRepository roadConditionRepository; + @Autowired + AccidentRepository repository; + + @Autowired + AccidentService accidentService; + + @Value("${local.server.port}") + private int port; + + private URL base; + private RestTemplate template; + + static Logger log = Logger.getLogger(HomeWork5Test.class.getName()); + public static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + + @Before + public void init() throws Exception { + this.base = new URL("http://localhost:" + port + "/"); + template = new TestRestTemplate(); +// RestAssured.port = port; + } + + @Test + public void AcciedentsTest() { + Accidents acciedent = new Accidents("200901BS70001"); + log.info(acciedent); + + } + + @Test + public void AccidentRepositoryTest() { + String testId = "200901BS70004"; + assertThat(repository.count(), equalTo(16L)); + Accidents accidents = repository.findOne(testId); + log.info(accidents); + assertThat(accidents.getId(), equalTo(testId)); + } + + @Test + public void getHello() throws Exception { + ResponseEntity response = template.getForEntity(base.toString() + "hello/Bill", String.class); + assertThat(response.getBody(), equalTo("hello, Bill")); + } + + @Test + public void getAllAccidentsByRoadConditionTest() { + RoadConditions roadCondition = roadConditionRepository.findOne(2); + List accidentsList = repository.findByRoadSurfaceConditions(roadCondition); + log.info(accidentsList); + assertThat(accidentsList.size(), equalTo(4)); + assertThat(accidentsList.get(0).getRoadSurfaceConditions().getCode(), equalTo(roadCondition.getCode())); + } + + @Test + // Use default countBy query + public void getAllAccidentsGroupByRoadCondition1Test() { + List roadConditionsList = roadConditionRepository.findAll(); + for (RoadConditions roadCondition : roadConditionsList) { + log.info(roadCondition.getCode() + " , " + roadCondition.getLabel() + " , Count=" + + repository.countByRoadSurfaceConditions(roadCondition)); + } + assertThat(roadConditionsList.size(), equalTo(8)); + } + + @Test + // Use accident service + public void getAllAccidentsGroupByRoadCondition2Test() { + Map roadConditionsList = accidentService.getAccidentCountGroupByRoadCondition(); + // Java8 + roadConditionsList.forEach((k, v) -> log.info("Road Condition: " + k + ", Count=" + v)); + assertThat(roadConditionsList.get("Dry"), equalTo(11)); + } + + @Test + public void getAllAccidentsByDateTest() throws ParseException { + RoadConditions roadCondition = roadConditionRepository.findOne(1); + List accidentsList = repository.findByRoadSurfaceConditionsAndDateBetween( + roadCondition, + simpleDateFormat.parse("2009-01-01"), + simpleDateFormat.parse("2009-01-10")); + log.info(accidentsList); + assertThat(accidentsList.size(), equalTo(8)); + } + + @Test + public void getAllAccidentsByWeatherConditionAndYearTest() { + Map weatherConditionsList = accidentService.getAccidentCountGroupByWeatherConditionAndYear("2009"); + weatherConditionsList.forEach((k, v) -> log.info("Weather Condition: " + k + ", Count=" + v)); + assertThat(weatherConditionsList.get("Fine no high winds"), equalTo(12)); + } + + @Test + public void updateAccidentByDate() throws ParseException { + String testDate = "2009-01-07"; + int count = accidentService.updateAccidentTimeByDate(testDate); + assertThat(count, equalTo(2)); + List accidentsList = repository.findByDate(simpleDateFormat.parse(testDate)); + log.info(accidentsList); + assertThat(accidentsList.get(0).getTime(), equalTo("AFTERNOON")); + assertThat(accidentsList.get(1).getTime(), equalTo("NIGHT")); + } + + @Test + public void RoadConditionRepositoryTest() { + assertThat(roadConditionRepository.count(), equalTo(8L)); + List roadConditionsList = roadConditionRepository.findAll(); + log.info(roadConditionsList); + assertThat(roadConditionsList.size(), equalTo(8)); + } + +} +/* + +Scenarios to be implemented for Homework: + + + 1. Find all the accidents by ID(Note: We can use findOne method which will accept the Accident ID as PK). + + 2. Find all the accidents count groupby all roadsurface conditions . + + 3. Find all the accidents count groupby accident year and weather condition .( For eg: in year 2009 we need to know the number of accidents based on each weather condition). + + 4. On a given date, fetch all the accidents and update the Time based on the below rules + + Time Logic: + MORNING - 6 am to 12 pm + AFTERNOON - 12 pm to 6 pm + EVENING - 6 pm to 12 am + NIGHT - 12 am to 6 am + */ \ No newline at end of file diff --git a/src/test/java/com/epam/dataservice/HomeWork6Test.java b/src/test/java/com/epam/dataservice/HomeWork6Test.java new file mode 100644 index 0000000..412093a --- /dev/null +++ b/src/test/java/com/epam/dataservice/HomeWork6Test.java @@ -0,0 +1,150 @@ +package com.epam.dataservice; + +import com.epam.springboot.AccidentsRestApplication; +import com.epam.springboot.WeatherUnitTest; +import com.epam.springboot.controller.AccidentController; +import com.epam.springboot.modal.Accidents; +import com.epam.springboot.modal.RoadConditions; +import com.epam.springboot.modal.WeatherConditions; +import com.github.springtestdbunit.DbUnitTestExecutionListener; +import com.github.springtestdbunit.annotation.DatabaseSetup; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.SpringApplicationConfiguration; +import org.springframework.http.MediaType; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyString; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * Created by bill on 16-5-22. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringApplicationConfiguration(classes = AccidentsRestApplication.class) //MockServletContext +@TestExecutionListeners({DependencyInjectionTestExecutionListener.class, DbUnitTestExecutionListener.class}) +@WebAppConfiguration +@DatabaseSetup("/sampleData.xml") +public class HomeWork6Test { + + @Autowired + AccidentController accidentController; + + static Logger log = Logger.getLogger(HomeWork6Test.class.getName()); + private final static String REST_URL = "/accidents"; + private MockMvc mvc; + + @Before + public void setUp() throws Exception { + mvc = MockMvcBuilders.standaloneSetup(accidentController).build(); + } + + @Test + public void contextLoads() { + } + +/* + @Test + public void RoadConditionRepositoryTest() { + assertThat(roadConditionRepository.count(), equalTo(8L)); + List roadConditionsList = roadConditionRepository.findAll(); + log.info(roadConditionsList); + assertThat(roadConditionsList.size(), equalTo(8)); + } +*/ + + @Test + public void helloTest() throws Exception { + String name = "Bill"; + mvc.perform(MockMvcRequestBuilders.get("/hello/" + name).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(equalTo("hello, " + name))); + } + + @Test + public void accidentAllTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.get("/accidents").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("2009-01"))); + } + + @Test + public void accidentOneTest() throws Exception { + String test_id = "200901BS70016"; + mvc.perform(MockMvcRequestBuilders.get(REST_URL + "/" + test_id).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(test_id))); + } + + @Test + public void createAccidentTest() throws Exception { + //"{"id":"200901BS70001","weatherConditions":{"code":6,"label":"Snowing + high winds"},"roadSurfaceConditions":{"code":3,"label":"Snow"},"date":"2009-01-01","time":"15:11","longitude":-0.201349,"latitude":51.512273,"policeForce":1,"severity":2,"numberOfVehicles":2,"numberOfCasualties":1,"dayOfWeek":5,"districtAuthority":12,"lightConditions":1}" + String test_id = "201606BS70008"; + Accidents accidents = new Accidents(test_id, new WeatherConditions(6), new RoadConditions(3), new java.util.Date()); + mvc.perform(MockMvcRequestBuilders.post(REST_URL) + .contentType(MediaType.APPLICATION_JSON) + .content(WeatherUnitTest.toJsonBytes(accidents)) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(accidents.getId()))); + mvc.perform(MockMvcRequestBuilders.get(REST_URL + "/" + test_id).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Snow"))); + } + + @Test + public void updateAccidentTest() throws Exception { + String test_id = "200901BS70004"; + String url = REST_URL + "/" + test_id; + String json = "{\"id\":\"" + test_id + "\",\"dayOfWeek\":5}"; + Accidents accidents = new Accidents(test_id); + mvc.perform(MockMvcRequestBuilders.put(url) + .contentType(MediaType.APPLICATION_JSON) + .content(json) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(accidents.getId()))); + mvc.perform(MockMvcRequestBuilders.get(url).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("\"dayOfWeek\":5"))); + } + + @Test + public void deleteAccidentTest() throws Exception { + String test_id = "200901BS70004"; + String url = REST_URL + "/" + test_id; + mvc.perform(MockMvcRequestBuilders.get(url).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(test_id))); + mvc.perform(MockMvcRequestBuilders.delete(url).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + mvc.perform(MockMvcRequestBuilders.get(url).accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(isEmptyString())); + } + + +} +/* +//todo add Restful services to get all accidents +//todo add Restful services to get accident by Id +//todo add Restful services to get accident by day +//todo add Restful services to create new accident (with POST and PUT - you do remember difference, right? ;) ) +//todo add Restful services to update existed accident +//todo add Restful services to delete accident +//todo for put method response 404(Not Found) when encounter the exception that the id is not exist​ + + */ \ No newline at end of file diff --git a/src/test/java/com/epam/processor/DataProcessorTest.java b/src/test/java/com/epam/processor/DataProcessorTest.java index de1b3e8..9199257 100644 --- a/src/test/java/com/epam/processor/DataProcessorTest.java +++ b/src/test/java/com/epam/processor/DataProcessorTest.java @@ -57,13 +57,13 @@ public void should_return_null_if_no_such_index(){ @Test public void should_count_by_road_conditions7(){ Map countByRoadConditions = dataProcessor.getCountByRoadSurfaceCondition7(); - assertThat(countByRoadConditions.get("Dry"), is(110277)); + assertThat(countByRoadConditions.get("Dry"), is(110277L)); } @Test public void should_count_by_road_conditions(){ Map countByRoadConditions = dataProcessor.getCountByRoadSurfaceCondition(); - assertThat(countByRoadConditions.get("Dry"), is(110277)); + assertThat(countByRoadConditions.get("Dry"), is(110277L)); } @Test diff --git a/src/test/java/com/epam/processor/DbPrepare.java b/src/test/java/com/epam/processor/DbPrepare.java new file mode 100644 index 0000000..1097d15 --- /dev/null +++ b/src/test/java/com/epam/processor/DbPrepare.java @@ -0,0 +1,90 @@ +package com.epam.processor; + +import org.dbunit.dataset.DataSetException; +import org.dbunit.dataset.IDataSet; +import org.dbunit.dataset.ITable; +import org.dbunit.dataset.ITableIterator; +import org.dbunit.dataset.csv.CsvDataSet; +import org.dbunit.dataset.xml.FlatXmlWriter; +import org.dbunit.dataset.xml.XmlDataSet; +import org.dbunit.util.TableFormatter; +import org.dbunit.util.fileloader.CsvDataFileLoader; +import org.dbunit.util.fileloader.DataFileLoader; +import org.junit.Test; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Created by bill on 16-5-25. + */ +public class DbPrepare { + public static String CSV_FILE_PATH = Paths.get("src","main","resources").toString(); + public static String DB_DATA_PATH = Paths.get("src","test","resources").toString(); + public static Path table_xml = Paths.get(DB_DATA_PATH,"tables.xml"); + public static Path sampleData_xml = Paths.get(DB_DATA_PATH,"sampleData.xml"); + + public void dumpTables(IDataSet dataSet) { + try { + ITableIterator tableIterator = dataSet.iterator(); + while (tableIterator.next()) { + ITable table = tableIterator.getTable(); + System.out.println(new TableFormatter().format(table)); + } + } catch (DataSetException e) { + e.printStackTrace(); + } + } + + @Test + public void csvToXml() { + DataFileLoader loader = new CsvDataFileLoader(); + String[] tables = new String[0]; + try { + IDataSet dataSet = loader.load(CSV_FILE_PATH); + tables = dataSet.getTableNames(); + } catch (DataSetException e) { + e.printStackTrace(); + } + System.out.println(tables); + } + + @Test + public void flatXmlDataSetTest() { +// IDataSet dataSet = new FlatXmlDataSetLoader(new FileInputStream(table_xml.toString())); + } + @Test + public void csvToFlatXml() throws Exception { + IDataSet dataSet = new CsvDataSet(new File(CSV_FILE_PATH)); +// dumpTables(dataSet); + System.out.println("Writing to " + sampleData_xml.toString()); + FileOutputStream fos = new FileOutputStream(sampleData_xml.toString()); + FlatXmlWriter flatXmlWriter = new FlatXmlWriter(fos); + flatXmlWriter.write(dataSet); + } + + @Test + public void loadXml() { + try { + System.out.println("Loading " + table_xml.toString()); + IDataSet dataSet2 = new XmlDataSet(new FileInputStream(table_xml.toString())); + dumpTables(dataSet2); + } catch (IOException e) { + e.printStackTrace(); + } catch (DataSetException e) { + e.printStackTrace(); + } + } + + @Test + public void csvToDbunitXml() throws Exception { + IDataSet dataSet = new CsvDataSet(new File(CSV_FILE_PATH)); + dumpTables(dataSet); + FileOutputStream fos = new FileOutputStream(sampleData_xml.toString()); + FlatXmlWriter flatXmlWriter = new FlatXmlWriter(fos); +// XmlDataSetWriter xmlWriter = new XmlDataSetWriter(fos, null); +// xmlWriter.write(dataSet); + flatXmlWriter.write(dataSet); + } +} diff --git a/src/test/java/com/epam/springboot/WeatherUnitTest.java b/src/test/java/com/epam/springboot/WeatherUnitTest.java new file mode 100644 index 0000000..bbfd388 --- /dev/null +++ b/src/test/java/com/epam/springboot/WeatherUnitTest.java @@ -0,0 +1,138 @@ +package com.epam.springboot; + +import com.epam.springboot.controller.WeatherController; +import com.epam.springboot.modal.WeatherConditions; +import com.epam.springboot.repository.WeatherConditionRepository; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.springtestdbunit.DbUnitTestExecutionListener; +import com.github.springtestdbunit.annotation.DatabaseSetup; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.SpringApplicationConfiguration; +import org.springframework.boot.test.WebIntegrationTest; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockServletContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * Created by bill on 16-5-29. + */ +@RunWith(SpringJUnit4ClassRunner.class) +@SpringApplicationConfiguration(classes = AccidentsRestApplication.class) //MockServletContext +//@ContextConfiguration(classes = MockServletContext.class) +@TestExecutionListeners({DependencyInjectionTestExecutionListener.class, DbUnitTestExecutionListener.class}) +@WebAppConfiguration +//@WebIntegrationTest +//@IntegrationTest("server.port:8090") +@DatabaseSetup("/sampleData.xml") +//@WebMvcTest(WeatherController.class) +public class WeatherUnitTest { +// @Autowired +// WeatherConditionRepository weatherConditionRepository; + @Autowired + WeatherController weatherController; + + static Logger log = Logger.getLogger(WeatherUnitTest.class.getName()); + private MockMvc mvc; + private WeatherConditions sampleWatherConditions = new WeatherConditions(20,"Test20 Sample"); + + @Before + public void setUp() throws Exception { + mvc = MockMvcBuilders.standaloneSetup(weatherController).build(); + } + + @Test + public void simpleWeatherTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.get("/weather/test").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Test20"))); + } + + @Test + public void weatherAllTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.get("/weather").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()); + } + + @Test + public void weatherOneTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.get("/weather/3").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Snowing"))); + } + + @Test + // curl -X POST -H "Content-Type:application/json" -d '{ "code": "20", "label":"Test20" }' http://localhost:8080/weather + public void weatherCreateTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.post("/weather") + .contentType(MediaType.APPLICATION_JSON) + .content(toJsonBytes(sampleWatherConditions)) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(sampleWatherConditions.getLabel()))); + } + + @Test + // curl -X PUT -H "Content-Type:application/json" -d '{ "code": "8" }' http://localhost:8080/weather/8 + public void weatherPutTest() throws Exception { + WeatherConditions weatherConditions = new WeatherConditions(8,"Updated info"); + log.info("Update /weather/"+weatherConditions.getCode()); + mvc.perform(MockMvcRequestBuilders.put("/weather/"+weatherConditions.getCode()) + .contentType(MediaType.APPLICATION_JSON) + .content(toJsonBytes(weatherConditions)) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString(weatherConditions.getLabel()))); + } + + @Test + // curl -X DELETE http://localhost:8080/weather/3 + public void weatherDeleteTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.delete("/weather/3").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().string(containsString("Snowing"))); + } + + @Test + // curl -X PUT -H "Content-Type:application/json" -d '{ "code": "20" }' http://localhost:8080/weather/20 + public void weatherPutExceptionTest() throws Exception { + mvc.perform(MockMvcRequestBuilders.put("/weather/"+sampleWatherConditions.getCode()) + .contentType(MediaType.APPLICATION_JSON) + .content(toJsonBytes(sampleWatherConditions)) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().is4xxClientError()); + } + + + public static byte[] toJsonBytes(Object obj) { + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + try { + return mapper.writeValueAsBytes(obj); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + return null; + } + +} diff --git a/src/test/resources/sampleData.xml b/src/test/resources/sampleData.xml new file mode 100644 index 0000000..4e7141c --- /dev/null +++ b/src/test/resources/sampleData.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/tables.xml b/src/test/resources/tables.xml new file mode 100644 index 0000000..1020c3a --- /dev/null +++ b/src/test/resources/tables.xml @@ -0,0 +1,497 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +