Feat/#19 dictionaries tops

.gitignore

@@ -82,3 +82,9 @@ nb-configuration.xml
 ## OS X
 ##############################
 .DS_Store
+
+
+##############################
+## My private ignore directory, to be able to use temp files
+##############################
+.ignoreme/

.run/KgParserApplication.run.xml

@@ -2,7 +2,7 @@
   <configuration default="false" name="KgParserApplication" type="Application" factoryName="Application" nameIsGenerated="true">
     <option name="MAIN_CLASS_NAME" value="ru.klavogonki.kgparser.springboot.KgParserApplication" />
     <module name="kgparser-springboot" />
-    <option name="PROGRAM_PARAMETERS" value="C:\java\kg\home\ec2-user\kg 20001 625000 &quot;2020-12-08 02-39-07&quot;" />
+    <option name="PROGRAM_PARAMETERS" value="d:/kg/2020.12.28 1 628000 1 &quot;2020-12-28 00-28-13&quot; " />
     <extension name="coverage">
       <pattern>
         <option name="PATTERN" value="ru.klavogonki.kgparser.export.*" />

.run/PlayerDataDownloader.run.xml

@@ -5,7 +5,7 @@
     <option name="PROGRAM_PARAMETERS" value="$PROJECT_DIR$/../kg/ 100000 110000 100" />
     <extension name="coverage">
       <pattern>
-        <option name="PATTERN" value="ru.klavogonki.kgparser.jsonParser.entity.*" />
+        <option name="PATTERN" value="ru.klavogonki.kgparser.*" />
         <option name="ENABLED" value="true" />
       </pattern>
     </extension>

.run/PlayerJsonParser.run.xml

@@ -5,7 +5,7 @@
     <option name="PROGRAM_PARAMETERS" value="C:\java\kg\home\ec2-user\kg 1 625000 &quot;2020-12-08 02-39-07&quot;" />
     <extension name="coverage">
       <pattern>
-        <option name="PATTERN" value="ru.klavogonki.kgparser.jsonParser.entity.*" />
+        <option name="PATTERN" value="ru.klavogonki.kgparser.*" />
         <option name="ENABLED" value="true" />
       </pattern>
     </extension>

kgparserSpringBoot/pom.xml

@@ -52,6 +52,13 @@
 			<artifactId>spring-boot-starter-data-jpa</artifactId>
 		</dependency>
 
+		<!-- Enable javax.validation in SpringBoot -->
+		<!-- see https://nullbeans.com/how-to-use-java-bean-validation-in-spring-boot/ -->
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-validation</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>

kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/jsonParser/entity/PlayerEntity.java

@@ -1,36 +1,47 @@
 package ru.klavogonki.kgparser.jsonParser.entity;
 
 import lombok.Data;
+import org.hibernate.annotations.NaturalId;
 import ru.klavogonki.kgparser.Rank;
 import ru.klavogonki.kgparser.http.UrlConstructor;
 
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
+import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Embedded;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
 import javax.persistence.Id;
+import javax.persistence.OneToMany;
+import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
 import javax.persistence.Transient;
 import java.time.LocalDateTime;
+import java.util.List;
 
 @Data
 @Entity
 @Table(name = "Player")
 public class PlayerEntity {
 
     @Id
-    @GeneratedValue
+//    @GeneratedValue
+    @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "pl_SEQ")
+    @SequenceGenerator(name = "pl_SEQ", sequenceName = "pl_SEQ", allocationSize = 1000)
     private Long dbId;
 
     private LocalDateTime importDate; // when PlayerDataDownloader has been executed
 
     private String getSummaryError; // we import all users, including non-existing
 
-    private String getIndexDataError; // we import all users, including non-existing and users with failed /get-indexData
+    private String getIndexDataError; // we import all users, including non-existing and users with failed /get-index-data
+
+    private String getStatsOverviewError; // we import all users, including non-existing and users with failed /get-stats-overview
 
     // fields from PlayerSummary
+    @NaturalId
     private Integer playerId; // KG user id
 
     private String login;
@@ -75,6 +86,10 @@ public class PlayerEntity {
 
     private Integer carsCount;
 
+    @OneToMany(cascade = CascadeType.ALL)
+//    @JoinColumn(name = "player_id") // do not create a join table! // todo: it's hard for non-pk column, does not work easily. Investigate this
+    List<PlayerVocabularyStatsEntity> stats;
+
     @Transient
     public String getProfileLink() {
         return UrlConstructor.userProfileLink(playerId);

kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/jsonParser/entity/PlayerVocabularyStatsEntity.java

@@ -0,0 +1,104 @@
+package ru.klavogonki.kgparser.jsonParser.entity;
+
+import lombok.Data;
+import ru.klavogonki.kgparser.Dictionary;
+import ru.klavogonki.kgparser.DictionaryMode;
+import ru.klavogonki.kgparser.NonStandardDictionaryType;
+
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.JoinColumn;
+import javax.persistence.ManyToOne;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+import java.time.LocalDateTime;
+
+@Data
+@Entity
+@Table(name = "Player_Vocabulary_Stats")
+public class PlayerVocabularyStatsEntity {
+    @Id
+//    @GeneratedValue
+    @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "pvs_SEQ")
+    @SequenceGenerator(name = "pvs_SEQ", sequenceName = "pvs_SEQ", allocationSize = 1000)
+    private Long dbId;
+
+    private LocalDateTime importDate; // when PlayerDataDownloader has been executed
+
+    // Caused by: org.hibernate.DuplicateMappingException: Table [player_vocabulary_stats] contains physical column name [player_id] referred to by multiple logical column names: [player_id], [playerId]
+    // private Integer playerId; // KG user id
+
+    @ManyToOne
+//    @JoinColumn(name = "player_id", referencedColumnName = "player_id")
+    @JoinColumn(nullable = false, name = "player_id", referencedColumnName = "playerId") // weird behaviour, db column name fails, entity field name is working.
+    private PlayerEntity player;
+
+    private String error; // also duplicated in PlayerEntity#getStatsOverviewError
+
+    // todo: this can be extracted to a separate VocabularyEntity
+
+    private String vocabularyCode;
+
+    private Integer vocabularyId; // for non-standard dictionaries only
+
+    private Integer vocabularyInfoId; // probably the db id from KG database, gametype[i].info.id
+
+    private String vocabularyName;
+
+    @Enumerated(EnumType.STRING)
+    private NonStandardDictionaryType vocabularyType;
+
+    private Integer vocabularySymbols; // for non-standard dictionaries only
+
+    private Integer vocabularyRows; // for non-standard dictionaries only
+
+    /**
+     * <ul>
+     *  <li>Для Обычного, Букв, Абракадабры, Яндекс.Рефератов, Цифр — {@code normal}</li>
+     *  <li>Для Безошибочного — {@code noerror}</li>
+     *  <li>Для Спринта — {@code sprint}</li>
+     *  <li>Для Марафона — {@code marathon}</li>
+     *  <li>Для нестандартных словарей — {@code normal}</li>
+     * </ul>
+     *
+     * @see DictionaryMode#getDictionaryMode
+     */
+    @Enumerated(EnumType.STRING)
+    private DictionaryMode vocabularyMode;
+
+    /**
+     * <ul>
+     *  <li>Для Обычного, Безошибочного, Спринта, Марафона — {@code 0}</li>
+     *  <li>Для Абракадабры — {@code -1}</li>
+     *  <li>Для Цифр — {@code -2}</li>
+     *  <li>Для Яндекс.Рефератов — {@code -3}</li>
+     *  <li>Для Букв — {@code -4}</li>
+     *  <li>Для нестандартных словарей — числовой id словаря.</li>
+     * </ul>
+     *
+     * @see Dictionary#getTextType
+     */
+    private Integer vocabularyTextType;
+
+    private Integer racesCount; // пробег игрока по словарю
+
+    private Double averageSpeed; // средняя скорость игрока по словарю, знаков в минуту
+
+    private Integer bestSpeed; // рекорд игрока по словарю, знаков в минуту
+
+    private Double averageError; // процент ошибок игрока по словарю, в процентах
+
+    private Integer haul; // время, проведённое игроком в словаре, в секундах
+
+    private Integer qual; // на сколько пройдена квалификация по словарю. Рекорды в словаре засчитываются в пределах 1.2 * qual.
+
+    private Integer dirty; // todo: wtf is this?
+
+    private LocalDateTime updated; // Время апдейта результата игроков по словарю. Московское время.
+
+    private Boolean bookDone; // пройдена ли книга. Заполнено только для словарей-книг.
+}

kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/jsonParser/mapper/PlayerMapper.java

@@ -3,9 +3,9 @@
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
 import ru.klavogonki.kgparser.PlayerJsonData;
-import ru.klavogonki.kgparser.jsonParser.PlayerIndexData;
 import ru.klavogonki.kgparser.jsonParser.entity.PlayerEntity;
 import ru.klavogonki.kgparser.util.DateUtils;
+import ru.klavogonki.openapi.model.Microtime;
 
 import java.time.LocalDateTime;
 
@@ -14,7 +14,7 @@ public interface PlayerMapper {
 
     // todo: add importDate?
     /* Yes, MapStruct works with public fields - a great library! */
-    // PlayerSummary fields
+    // GetSummaryResponse fields
     @Mapping(source = "summary.err", target = "getSummaryError")
     @Mapping(source = "summary.user.id", target = "playerId")
     @Mapping(source = "summary.user.login", target = "login")
@@ -24,19 +24,24 @@ public interface PlayerMapper {
     @Mapping(source = "summary.title", target = "title")
     @Mapping(source = "summary.blocked", target = "blocked")
 
-    // PlayerIndexData fields
+    // GetIndexDataResponse fields
     @Mapping(source = "indexData.err", target = "getIndexDataError")
     @Mapping(source = "indexData.stats.registered", target = "registered")
-    @Mapping(source = "indexData.stats.achievementsCount", target = "achievementsCount")
-    @Mapping(source = "indexData.stats.totalRacesCount", target = "totalRacesCount")
+    @Mapping(source = "indexData.stats.achievesCnt", target = "achievementsCount")
+    @Mapping(source = "indexData.stats.totalNumRaces", target = "totalRacesCount")
     @Mapping(source = "indexData.stats.bestSpeed", target = "bestSpeed")
     @Mapping(source = "indexData.stats.ratingLevel", target = "ratingLevel")
-    @Mapping(source = "indexData.stats.friendsCount", target = "friendsCount")
-    @Mapping(source = "indexData.stats.vocabulariesCount", target = "vocabulariesCount")
-    @Mapping(source = "indexData.stats.carsCount", target = "carsCount")
+    @Mapping(source = "indexData.stats.friendsCnt", target = "friendsCount")
+    @Mapping(source = "indexData.stats.vocsCnt", target = "vocabulariesCount")
+    @Mapping(source = "indexData.stats.carsCnt", target = "carsCount")
+
+    // GetStatsOverviewResponse fields
+    @Mapping(source = "statsOverview.err", target = "getStatsOverviewError")
+    // other fields will be parsed to PlayerVocabularyStatsEntity in a separate parser
+
     PlayerEntity playerJsonDataToPlayerEntity(PlayerJsonData data);
 
-    default LocalDateTime registeredToLocalDateTime(PlayerIndexData.Registered registered) {
+    default LocalDateTime registeredToLocalDateTime(Microtime registered) {
         return DateUtils.convertUserRegisteredTime(registered);
     }
 }

kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/jsonParser/mapper/PlayerVocabularyStatsMapper.java

@@ -0,0 +1,93 @@
+package ru.klavogonki.kgparser.jsonParser.mapper;
+
+import org.mapstruct.AfterMapping;
+import org.mapstruct.Context;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.mapstruct.MappingTarget;
+import org.mapstruct.ValueMapping;
+import org.mapstruct.ValueMappings;
+import ru.klavogonki.kgparser.DictionaryMode;
+import ru.klavogonki.kgparser.NonStandardDictionaryType;
+import ru.klavogonki.kgparser.jsonParser.entity.PlayerEntity;
+import ru.klavogonki.kgparser.jsonParser.entity.PlayerVocabularyStatsEntity;
+import ru.klavogonki.kgparser.util.DateUtils;
+import ru.klavogonki.openapi.model.GetStatsOverviewGameType;
+import ru.klavogonki.openapi.model.GetStatsOverviewResponse;
+import ru.klavogonki.openapi.model.NonStandardVocabularyType;
+import ru.klavogonki.openapi.model.VocabularyMode;
+
+import java.time.LocalDateTime;
+
+@Mapper
+public interface PlayerVocabularyStatsMapper {
+
+//    @Mapping(source = "", target = "vocabularyCode")
+    @Mapping(source = "id", target = "vocabularyId")
+    @Mapping(source = "name", target = "vocabularyName")
+    @Mapping(source = "numRaces", target = "racesCount")
+    @Mapping(source = "type", target = "vocabularyType")
+    @Mapping(source = "symbols", target = "vocabularySymbols")
+    @Mapping(source = "rows", target = "vocabularyRows")
+    @Mapping(source = "bookDone", target = "bookDone")
+
+    @Mapping(source = "info.id", target = "vocabularyInfoId")
+//    @Mapping(source = "info.userId", target = "playerId") // todo: map to PlayerEntity
+    @Mapping(source = "info.mode", target = "vocabularyMode")
+    @Mapping(source = "info.texttype", target = "vocabularyTextType")
+//    @Mapping(source = "info.numRaces", target = "") // we assume gameType.numRaces is the same, it is also validated
+    @Mapping(source = "info.avgSpeed", target = "averageSpeed")
+    @Mapping(source = "info.bestSpeed", target = "bestSpeed")
+    @Mapping(source = "info.avgError", target = "averageError")
+    @Mapping(source = "info.haul", target = "haul")
+    @Mapping(source = "info.qual", target = "qual")
+    @Mapping(source = "info.dirty", target = "dirty")
+    @Mapping(source = "info.updated", target = "updated", dateFormat = DateUtils.DATE_TIME_FORMAT_FOR_UI)
+    PlayerVocabularyStatsEntity statsGameTypeToEntity(
+        GetStatsOverviewGameType gameType,
+        @Context LocalDateTime importDate,
+        @Context GetStatsOverviewResponse response,
+        @Context String vocabularyCode,
+        @Context PlayerEntity player
+    );
+
+    @ValueMappings({
+        @ValueMapping(source = "WORDS", target = "words"),
+        @ValueMapping(source = "PHRASES", target = "phrases"),
+        @ValueMapping(source = "TEXTS", target = "texts"),
+        @ValueMapping(source = "URL", target = "url"),
+        @ValueMapping(source = "BOOK", target = "book"),
+        @ValueMapping(source = "GENERATOR", target = "generator"),
+    })
+    NonStandardDictionaryType nonStandardDictionaryType(NonStandardVocabularyType vocabularyType);
+
+    @ValueMappings({
+        @ValueMapping(source = "NORMAL", target = "normal"),
+        @ValueMapping(source = "NOERROR", target = "noerror"),
+        @ValueMapping(source = "SPRINT", target = "sprint"),
+        @ValueMapping(source = "MARATHON", target = "marathon"),
+    })
+    DictionaryMode dictionaryMode(VocabularyMode vocabularyMode);
+
+    @AfterMapping
+    default void fillContextData(
+        GetStatsOverviewGameType gameType,
+        @MappingTarget PlayerVocabularyStatsEntity entity,
+        @Context LocalDateTime importDate,
+        @Context GetStatsOverviewResponse response,
+        @Context String vocabularyCode,
+        @Context PlayerEntity player
+    ) {
+        entity.setImportDate(importDate);
+        entity.setError(response.getErr());
+        entity.setVocabularyCode(vocabularyCode);
+
+/*
+        // todo: do we need to set the whole playerEntity, at least with its dbId?
+        PlayerEntity playerEntity = new PlayerEntity();
+        playerEntity.setPlayerId(playerId);
+
+*/
+        entity.setPlayer(player);
+    }
+}

kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/jsonParser/repository/PlayerVocabularyStatsRepository.java

@@ -0,0 +1,7 @@
+package ru.klavogonki.kgparser.jsonParser.repository;
+
+import org.springframework.data.repository.CrudRepository;
+import ru.klavogonki.kgparser.jsonParser.entity.PlayerVocabularyStatsEntity;
+
+public interface PlayerVocabularyStatsRepository extends CrudRepository<PlayerVocabularyStatsEntity, Long> {
+}