feat: #81 Convert StatisticsGeneratorConfig to a simple Kotlin data c…

…lass. Statistics generation is working locally.
kghackers · Nov 21, 2024 · 4922d7d · 4922d7d
1 parent fb8395b
commit 4922d7d
Show file tree

Hide file tree

Showing 6 changed files with 149 additions and 71 deletions.
diff --git a/kgstatsSrv/src/main/java/ru/klavogonki/statistics/export/StatisticsGenerator.java b/kgstatsSrv/src/main/java/ru/klavogonki/statistics/export/StatisticsGenerator.java
@@ -120,31 +120,31 @@ public void generateStatistics(final Config config, final StatisticsGeneratorCon
         logger.debug("{}: {}", StatisticsGeneratorConfig.class.getSimpleName(), generatorConfig);
 
         // global tops exporters
-        export(context, generatorConfig.isExportIndexPage(), indexPageExporter);
-        export(context, generatorConfig.isExportTopBySpeed(), topBySpeedExporter);
-        export(context, generatorConfig.isExportTop500Pages(), top500PagesExporter);
-        export(context, generatorConfig.isExportPlayersByRank(), playersByRankExporter);
+        export(context, generatorConfig.getExportIndexPage(), indexPageExporter);
+        export(context, generatorConfig.getExportTopBySpeed(), topBySpeedExporter);
+        export(context, generatorConfig.getExportTop500Pages(), top500PagesExporter);
+        export(context, generatorConfig.getExportPlayersByRank(), playersByRankExporter);
 
         // standard vocabularies exporters
-        export(context, generatorConfig.isExportNormalTop(), normalTopExporter);
-        export(context, generatorConfig.isExportAbraTop(), abraTopExporter);
-        export(context, generatorConfig.isExportReferatsTop(), referatsTopExporter);
-        export(context, generatorConfig.isExportNoErrorTop(), noErrorTopExporter);
-        export(context, generatorConfig.isExportMarathonTop(), marathonTopExporter);
-        export(context, generatorConfig.isExportCharsTop(), charsTopExporter);
-        export(context, generatorConfig.isExportDigitsTop(), digitsTopExporter);
-        export(context, generatorConfig.isExportSprintTop(), sprintTopExporter);
+        export(context, generatorConfig.getExportNormalTop(), normalTopExporter);
+        export(context, generatorConfig.getExportAbraTop(), abraTopExporter);
+        export(context, generatorConfig.getExportReferatsTop(), referatsTopExporter);
+        export(context, generatorConfig.getExportNoErrorTop(), noErrorTopExporter);
+        export(context, generatorConfig.getExportMarathonTop(), marathonTopExporter);
+        export(context, generatorConfig.getExportCharsTop(), charsTopExporter);
+        export(context, generatorConfig.getExportDigitsTop(), digitsTopExporter);
+        export(context, generatorConfig.getExportSprintTop(), sprintTopExporter);
 
         // non-standard vocabularies exporters
-        export(context, generatorConfig.isExportNormalInEnglishTop(), normalInEnglishTopExporter);
-        export(context, generatorConfig.isExportMiniMarathonTop(), miniMarathonTopExporter);
-        export(context, generatorConfig.isExportShortTextsTop(), shortTextsTopExporter);
-        export(context, generatorConfig.isExportFrequencyVocabularyTop(), frequencyVocabularyTopExporter);
-        export(context, generatorConfig.isExportOneHundredRussianTop(), oneHundredRussianTopExporter);
-        export(context, generatorConfig.isExportDigitsOneHundredTop(), digitsOneHundredTopExporter);
-        export(context, generatorConfig.isExportTrainingIndexFingersTop(), trainingIndexFingersTopExporter);
-        export(context, generatorConfig.isExportRingFingersTop(), ringFingersTopExporter);
-        export(context, generatorConfig.isExportPinkiesPlusTop(), pinkiesPlusTopExporter);
+        export(context, generatorConfig.getExportNormalInEnglishTop(), normalInEnglishTopExporter);
+        export(context, generatorConfig.getExportMiniMarathonTop(), miniMarathonTopExporter);
+        export(context, generatorConfig.getExportShortTextsTop(), shortTextsTopExporter);
+        export(context, generatorConfig.getExportFrequencyVocabularyTop(), frequencyVocabularyTopExporter);
+        export(context, generatorConfig.getExportOneHundredRussianTop(), oneHundredRussianTopExporter);
+        export(context, generatorConfig.getExportDigitsOneHundredTop(), digitsOneHundredTopExporter);
+        export(context, generatorConfig.getExportTrainingIndexFingersTop(), trainingIndexFingersTopExporter);
+        export(context, generatorConfig.getExportRingFingersTop(), ringFingersTopExporter);
+        export(context, generatorConfig.getExportPinkiesPlusTop(), pinkiesPlusTopExporter);
     }
 
     // todo: think about moving StatisticsGeneratorConfig field determination to Exporter interface

diff --git a/kgstatsSrv/src/main/java/ru/klavogonki/statistics/export/StatisticsGeneratorConfig.java b/kgstatsSrv/src/main/java/ru/klavogonki/statistics/export/StatisticsGeneratorConfig.java
diff --git a/kgstatsSrv/src/main/kotlin/ru/klavogonki/statistics/export/StatisticsGeneratorConfig.kt b/kgstatsSrv/src/main/kotlin/ru/klavogonki/statistics/export/StatisticsGeneratorConfig.kt
@@ -0,0 +1,37 @@
+package ru.klavogonki.statistics.export
+
+/**
+ * Конфиг, настраивающий генерацию словарей.
+ *
+ * todo: по факту, в конфиг можно перенести настройки, которые сейчас реализуют классы,
+ * реализующие [ru.klavogonki.statistics.export.vocabulary.VocabularyTopExporter].
+ */
+data class StatisticsGeneratorConfig(
+    // aggregate tops
+    val exportIndexPage: Boolean = false,
+
+    val exportTopBySpeed: Boolean = false, // old exporter, not by normal vocabulary
+    val exportTop500Pages: Boolean = false,
+    val exportPlayersByRank: Boolean = false,
+
+    // standard vocabularies
+    val exportNormalTop: Boolean = false,
+    val exportAbraTop: Boolean = false,
+    val exportReferatsTop: Boolean = false,
+    val exportNoErrorTop: Boolean = false,
+    val exportMarathonTop: Boolean = false,
+    val exportCharsTop: Boolean = false,
+    val exportDigitsTop: Boolean = false,
+    val exportSprintTop: Boolean = false,
+
+    // non-standard vocabularies
+    val exportNormalInEnglishTop: Boolean = false,
+    val exportMiniMarathonTop: Boolean = false,
+    val exportShortTextsTop: Boolean = false,
+    val exportFrequencyVocabularyTop: Boolean = false,
+    val exportOneHundredRussianTop: Boolean = false,
+    val exportDigitsOneHundredTop: Boolean = false,
+    val exportTrainingIndexFingersTop: Boolean = false,
+    val exportRingFingersTop: Boolean = false,
+    val exportPinkiesPlusTop: Boolean = false
+)
diff --git a/kgstatsSrv/src/main/resources/application.actions.properties b/kgstatsSrv/src/main/resources/application.actions.properties
@@ -1,7 +1,9 @@
 spring.datasource.continue-on-error=true
 
 spring.jpa.hibernate.ddl-auto=update
-spring.datasource.url=jdbc:mysql://localhost:3306/actions?serverTimezone=UTC&rewriteBatchedStatements=true&profileSQL=false&useUnicode=yes&characterEncoding=UTF-8
+#spring.datasource.url=jdbc:mysql://localhost:3306/actions?serverTimezone=UTC&rewriteBatchedStatements=true&profileSQL=false&useUnicode=yes&characterEncoding=UTF-8
+spring.datasource.url=jdbc:mysql://localhost:3306/kgparser-2024.11.11?serverTimezone=UTC&rewriteBatchedStatements=true&profileSQL=false&useUnicode=yes&characterEncoding=UTF-8
+
 spring.datasource.username=root
 spring.datasource.password=root
 
@@ -15,4 +17,3 @@ spring.jpa.properties.hibernate.generate_statistics=false
 # log sql statements by Hibernate
 spring.jpa.show-sql=false
 spring.jpa.properties.hibernate.format_sql=true
-
diff --git a/run-statistics-copy-static-files.sh b/run-statistics-copy-static-files.sh
@@ -0,0 +1,86 @@
+echo "Arguments: <minPlayerId> <maxPlayerId> <threadsCount>"
+echo ""
+
+echo "Environment variables values: "
+echo "ROOT_WORKING_DIR: $ROOT_WORKING_DIR"
+echo "LOG4J_XML_FILE_PATH: $LOG4J_XML_FILE_PATH"
+echo "KGSTATS_SRV_JAR_FILE_PATH: $KGSTATS_SRV_JAR_FILE_PATH"
+echo "KGSTATS_SRV_SQL_DIR: $KGSTATS_SRV_SQL_DIR"
+echo "KGSTATS_WEB_ROOT_DIR: $KGSTATS_WEB_ROOT_DIR"
+echo "SPRING_CONFIG_LOCATION: $SPRING_CONFIG_LOCATION"
+echo "GENERATE_STATISTICS_CONFIG_LOCATION: $GENERATE_STATISTICS_CONFIG_LOCATION"
+echo "DATABASE_USER: $DATABASE_USER"
+echo "DATABASE_PASSWORD: $DATABASE_PASSWORD"
+echo "DATABASE_NAME: $DATABASE_NAME"
+echo ""
+
+# declare required variables
+# todo: set defaults, see https://stackoverflow.com/a/2013589/8534088
+# todo: we should be able to override them from the command line
+#ROOT_WORKING_DIR=/d/kg/
+
+#LOG4J_XML_FILE_PATH=log4j2.xml
+
+#KGSTATS_SRV_JAR_FILE_PATH=kgstatsSrv/target/kgstats-srv-1.0.jar
+# todo: maybe take kgstatsSrv statics from target?
+#KGSTATS_SRV_SQL_DIR=kgstatsSrv/src/main/resources/sql
+# todo: maybe take kgstatsWeb statics from target?
+#KGSTATS_WEB_ROOT_DIR=kgstatsWeb/src/main/webapp
+#SPRING_CONFIG_LOCATION=kgstatsSrv/src/main/resources/application.actions.properties
+
+#DATABASE_USER=root
+#DATABASE_PASSWORD=root
+#DATABASE_NAME=actions
+
+DATABASE_DUMP_FILE_NAME=database.sql
+DATABASE_DUMP_FILE_PATH=$ROOT_WORKING_DIR/$DATABASE_DUMP_FILE_NAME
+
+JSON_FILES_DIRECTORY_NAME=/json
+JSON_FILES_DIRECTORY_PATH=$ROOT_WORKING_DIR/$JSON_FILES_DIRECTORY_NAME
+
+INPUT_CONFIG_FILE_NAME=config.json
+INPUT_CONFIG_FILE_PATH=$ROOT_WORKING_DIR/$INPUT_CONFIG_FILE_NAME
+
+OUTPUT_CONFIG_FILE_NAME=config-output.json
+OUTPUT_CONFIG_FILE_PATH=$ROOT_WORKING_DIR/$OUTPUT_CONFIG_FILE_NAME
+
+GENERATE_STATISTICS_DIRECTORY_NAME=/stats
+GENERATE_STATISTICS_DIRECTORY_PATH=$ROOT_WORKING_DIR/$GENERATE_STATISTICS_DIRECTORY_NAME
+STATIC_DIR=$GENERATE_STATISTICS_DIRECTORY_PATH
+
+# zip files
+JSON_DATA_ZIP_FILE_NAME=json.zip
+STATISTICS_ZIP_FILE_NAME=statistics.zip
+DATABASE_DUMP_ZIP_FILE_NAME=database.zip
+
+# s3 buckets
+S3_BUCKET_TEST=klavostat-test
+S3_BUCKET_DATA=klavostat-data
+
+# create config json file from input arguments
+# todo: add input arguments validation, although it will already work in SpringBoot app
+MIN_PLAYER_ID=$1
+MAX_PLAYER_ID=$2
+THREADS_COUNT=$3
+
+# copy static files from sources of kgstatsWeb to generated statistics
+cp -R $KGSTATS_WEB_ROOT_DIR/css $STATIC_DIR
+cp -R $KGSTATS_WEB_ROOT_DIR/img $STATIC_DIR
+
+# make sure /js is a directory
+mkdir -p "$STATIC_DIR/js"
+
+# make sure /xlsx is a directory
+mkdir -p "$STATIC_DIR/xlsx"
+
+# todo: after we exclude the generated files from sources, copy js recursively
+
+# cp -R $KGSTATS_WEB_ROOT_DIR/js $STATIC_DIR
+cp "$KGSTATS_WEB_ROOT_DIR/js/players-by-rank-chart.js" "$STATIC_DIR/js"
+cp "$KGSTATS_WEB_ROOT_DIR/js/stats-data.js" "$STATIC_DIR/js"
+cp "$KGSTATS_WEB_ROOT_DIR/js/stats-top-table.js" "$STATIC_DIR/js"
+
+# todo: old tops must be copied from archive S3 buckets, not from sources
+cp -R $KGSTATS_WEB_ROOT_DIR/2020-12-09 $STATIC_DIR
+
+echo "Copied static files from $KGSTATS_WEB_ROOT_DIR to $STATIC_DIR."
diff --git a/run-statistics-process-my-windows-set-env.sh b/run-statistics-process-my-windows-set-env.sh
@@ -1,4 +1,5 @@
-export ROOT_WORKING_DIR=/d/kg/
+#export ROOT_WORKING_DIR=/d/kg/
+export ROOT_WORKING_DIR=/c/kg/2024.11.11
 
 export LOG4J_XML_FILE_PATH=./log4j2.xml