Add json file parsing into mysql #48

kghackers · Jan 10, 2021 · 267f406 · dmitry-weirdo · Jan 13, 2021 · dmitry-weirdo
1 parent 55d5d06
commit 267f406
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 27 deletions.
diff --git a/.github/workflows/download.yaml b/.github/workflows/download.yaml
@@ -89,4 +89,31 @@ jobs:
 
       # Upload Archived Data into Bucket
       - name: Copy Archived Data to S3
-        run: aws s3 cp "$(ls data).tar.gz" s3://dievri-kgparser
+        run: aws s3 cp "$(ls data).tar.gz" s3://dievri-kgparser
+
+      # Process downloaded files
+      - name: Prepare MySQL database
+        run: |
+              sudo systemctl start mysql
+              sudo mysql -uroot -proot -e "create database actions;"
+      
+      - name: Parse Json Files
+        run: java -jar kgparserSpringBoot/target/kgparser-springboot-1.0.jar "data" ${{ github.event.inputs.first_id }} ${{ github.event.inputs.last_id }} 1 "$(ls data)"
+
+      - name: Create Database Dump
+        run: sudo mysqldump -uroot -proot actions > "$(ls data).sql"
+
+      - name: Upload DB Dump to s3
+        run: aws s3 cp "$(ls data).sql" s3://dievri-kgparser
+
+      - name: Telegram Message Notify
+        uses: appleboy/[email protected]
+        with:
+          format: html
+          to: ${{ secrets.TGBOT_CHAT }}
+          token: ${{ secrets.TGBOT_TOKEN }}
+          message: |
+            <strong>GitHub Player Data Downloader (Manual Dispatch)</strong>
+            <pre><code>
+              archive with json files and database dump uploaded to s3
+            </code></pre>
diff --git a/kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/springboot/KgParserApplication.java b/kgparserSpringBoot/src/main/java/ru/klavogonki/kgparser/springboot/KgParserApplication.java
@@ -73,14 +73,14 @@ public static void main(String[] args) {
 	@Override
 	public void run(final String... args) {
 		// todo: parse context from args of from json file given by args
-		ExportContext context = new ExportContext();
-		context.webRootDir = "C:/java/kgparser/kgparserWeb/src/main/webapp/";
+		// ExportContext context = new ExportContext();
+		// context.webRootDir = "C:/java/kgparser/kgparserWeb/src/main/webapp/";
 
 		// data load from 2020-12-28
-		context.minPlayerId = 1;
-		context.maxPlayerId = 628000;
-		context.dataDownloadStartDate = DateUtils.parseLocalDateTimeWithUiDateFormat("2020-12-28 00:28:13");
-		context.dataDownloadEndDate = DateUtils.parseLocalDateTimeWithUiDateFormat("2020-12-28 01:44:43");
+		// context.minPlayerId = 1;
+		// context.maxPlayerId = 628000;
+		// context.dataDownloadStartDate = DateUtils.parseLocalDateTimeWithUiDateFormat("2020-12-28 00:28:13");
+		// context.dataDownloadEndDate = DateUtils.parseLocalDateTimeWithUiDateFormat("2020-12-28 01:44:43");
 
 /*
 		// data load from 2020-12-09
@@ -99,25 +99,25 @@ public void run(final String... args) {
 		}
 */
 
-		playersByRankExporter.export(context);
-		if (true) {
-			return;
-		}
-
-		top500PagesExporter.export(context);
-		if (true) {
-			return;
-		}
-
-		indexPageExporter.export(context);
-		if (true) {
-			return;
-		}
-
-		topBySpeedExporter.export(context);
-		if (true) {
-			return;
-		}
+		// playersByRankExporter.export(context);
+		// if (true) {
+		// 	return;
+		// }
+
+		// top500PagesExporter.export(context);
+		// if (true) {
+		// 	return;
+		// }
+
+		// indexPageExporter.export(context);
+		// if (true) {
+		// 	return;
+		// }
+
+		// topBySpeedExporter.export(context);
+		// if (true) {
+		// 	return;
+		// }
 
 
 		// todo: pass a path to a json file with config instead

diff --git a/kgparserSpringBoot/src/main/resources/application.properties b/kgparserSpringBoot/src/main/resources/application.properties
@@ -1,6 +1,6 @@
 spring.jpa.hibernate.ddl-auto=update
 #spring.datasource.url=jdbc:mysql://localhost:3306/kgparser?serverTimezone=UTC
-spring.datasource.url=jdbc:mysql://localhost:3306/kgparser_2020_12_28?serverTimezone=UTC&rewriteBatchedStatements=true&profileSQL=true
+spring.datasource.url=jdbc:mysql://localhost:3306/actions?serverTimezone=UTC&rewriteBatchedStatements=true&profileSQL=true
 spring.datasource.username=root
 spring.datasource.password=root