initial steam scrapper

promotion-notifier-backend/pn-site-scrapper/scrapper-app/src/contractTest/groovy/io/mkrzywanski/pn/scrapper/contract/BaseClass.groovy

@@ -1,6 +1,7 @@
 package io.mkrzywanski.pn.scrapper.contract
 
 import io.mkrzywanski.gpn.scrapper.domain.post.*
+import io.mkrzywanski.gpn.scrapper.domain.post.price.Currencies
 import io.mkrzywanski.pn.scrapper.app.adapters.publishing.QueuePostPublisher
 import io.mkrzywanski.pn.scrapper.app.infra.QueueConfig
 import org.springframework.amqp.rabbit.annotation.RabbitListener

promotion-notifier-backend/pn-site-scrapper/scrapper-app/src/main/java/io/mkrzywanski/pn/scrapper/app/adapters/persistance/GameOfferModel.java

@@ -1,7 +1,7 @@
 package io.mkrzywanski.pn.scrapper.app.adapters.persistance;
 
 import io.mkrzywanski.gpn.scrapper.domain.post.GameOffer;
-import io.mkrzywanski.gpn.scrapper.domain.post.GamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
 import lombok.Value;
 
 import java.util.UUID;

promotion-notifier-backend/pn-site-scrapper/scrapper-app/src/main/java/io/mkrzywanski/pn/scrapper/app/adapters/persistance/GamePriceToPriceType.java

@@ -1,6 +1,10 @@
 package io.mkrzywanski.pn.scrapper.app.adapters.persistance;
 
 import io.mkrzywanski.gpn.scrapper.domain.post.*;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.CompositeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.EmptyGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.FreeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
 
 import java.util.Map;
 import java.util.Optional;

promotion-notifier-backend/pn-site-scrapper/scrapper-app/src/main/java/io/mkrzywanski/pn/scrapper/app/adapters/persistance/PriceInfo.java

@@ -1,9 +1,9 @@
 package io.mkrzywanski.pn.scrapper.app.adapters.persistance;
 
-import io.mkrzywanski.gpn.scrapper.domain.post.CompositeGamePrice;
-import io.mkrzywanski.gpn.scrapper.domain.post.EmptyGamePrice;
-import io.mkrzywanski.gpn.scrapper.domain.post.FreeGamePrice;
-import io.mkrzywanski.gpn.scrapper.domain.post.GamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.CompositeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.EmptyGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.FreeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
 import io.mkrzywanski.gpn.scrapper.domain.post.NumberGamePrice;
 
 import java.math.BigDecimal;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/build.gradle

@@ -1,4 +1,9 @@
 dependencies {
     implementation group: 'org.jsoup', name: 'jsoup', version: '1.15.3'
+    implementation 'com.jayway.jsonpath:json-path:2.7.0'
     testImplementation group: 'com.github.tomakehurst', name: 'wiremock', version: "${wiremockVersion}"
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.4.2'
+    implementation group: 'com.google.guava', name: 'guava', version: '31.1-jre'
+
+
 }

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/gamehunter/GameHunterPriceParser.java

@@ -1,6 +1,10 @@
 package io.mkrzywanski.gpn.scrapper.domain.gamehunter;
 
-import io.mkrzywanski.gpn.scrapper.domain.post.*;
+import io.mkrzywanski.gpn.scrapper.domain.post.NumberGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.CompositeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.EmptyGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.FreeGamePrice;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
 
 import java.util.Collection;
 import java.util.HashSet;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/gamehunter/GameHunterScrappingService.java

@@ -48,36 +48,36 @@ public void scrap() {
             }
             LOGGER.info("Scrapped page {} . Posts {}", pageNumber, scrappedPosts);
             final Set<Hash> scrappedPostHashes = extractHashes(scrappedPosts);
-            final List<Hash> alreadySavedPosts = postRepository.findByHashIn(scrappedPostHashes);
+            final List<Hash> alreadySavedPostHashes = postRepository.findByHashIn(scrappedPostHashes);
 
             newPostHashes = new HashSet<>(scrappedPostHashes);
-            alreadySavedPosts.forEach(newPostHashes::remove);
+            alreadySavedPostHashes.forEach(newPostHashes::remove);
 
             LOGGER.info("New post hashes {}", newPostHashes);
-            LOGGER.info("Already saved post hashes {}", alreadySavedPosts);
+            LOGGER.info("Already saved post hashes {}", alreadySavedPostHashes);
 
             final List<Post> newPostsFromCurrentPage = scrappedPosts.stream()
-                    .filter(post -> !alreadySavedPosts.contains(post.getHash()))
+                    .filter(post -> !alreadySavedPostHashes.contains(post.getHash()))
                     .toList();
 
             LOGGER.info("New posts to be saved from page {} : {}", pageNumber, newPostsFromCurrentPage);
             allNewPosts.addAll(newPostsFromCurrentPage);
 
-            final boolean currentPageIsPartiallyScrapped = alreadySavedPosts.size() > 0;
+            final boolean currentPageIsPartiallyScrapped = alreadySavedPostHashes.size() > 0;
             if (currentPageIsPartiallyScrapped) {
                 break;
             }
 
             pageNumber++;
 
-        } while (true);
+        } while (!Thread.currentThread().isInterrupted());
 
         final Set<Post> distinctByHash = allNewPosts.stream()
                 .filter(distinctByKey(Post::getHash))
                 .collect(Collectors.toSet());
 
         postRepository.saveAll(distinctByHash);
-        postTransactionalOutboxRepository.put(new HashSet<>(distinctByHash));
+        postTransactionalOutboxRepository.put(distinctByHash);
     }
 
     private Set<Hash> extractHashes(final List<Post> scrappedPosts) {
@@ -95,7 +95,6 @@ private List<Post> scrapPage(final int pageNumber) {
     }
 
     private static <T> Predicate<T> distinctByKey(final Function<? super T, ?> keyExtractor) {
-
         final Map<Object, Boolean> seen = new ConcurrentHashMap<>();
         return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
     }

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/GameOffer.java

@@ -1,5 +1,6 @@
 package io.mkrzywanski.gpn.scrapper.domain.post;
 
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
 import lombok.Value;
 
 import java.util.UUID;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/NumberGamePrice.java

@@ -1,5 +1,8 @@
 package io.mkrzywanski.gpn.scrapper.domain.post;
 
+import io.mkrzywanski.gpn.scrapper.domain.post.price.Currencies;
+import io.mkrzywanski.gpn.scrapper.domain.post.price.GamePrice;
+
 import java.math.BigDecimal;
 import java.util.Currency;
 import java.util.Map;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/CompositeGamePrice.java → promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/price/CompositeGamePrice.java

@@ -1,4 +1,4 @@
-package io.mkrzywanski.gpn.scrapper.domain.post;
+package io.mkrzywanski.gpn.scrapper.domain.post.price;
 
 import java.math.BigDecimal;
 import java.util.Collection;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/Currencies.java → promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/price/Currencies.java

@@ -1,4 +1,4 @@
-package io.mkrzywanski.gpn.scrapper.domain.post;
+package io.mkrzywanski.gpn.scrapper.domain.post.price;
 
 import java.util.Currency;
 

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/EmptyGamePrice.java → promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/price/EmptyGamePrice.java

@@ -1,4 +1,4 @@
-package io.mkrzywanski.gpn.scrapper.domain.post;
+package io.mkrzywanski.gpn.scrapper.domain.post.price;
 
 import java.math.BigDecimal;
 import java.util.Collections;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/FreeGamePrice.java → promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/price/FreeGamePrice.java

@@ -1,4 +1,4 @@
-package io.mkrzywanski.gpn.scrapper.domain.post;
+package io.mkrzywanski.gpn.scrapper.domain.post.price;
 
 import java.math.BigDecimal;
 import java.util.Collections;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/GamePrice.java → promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/post/price/GamePrice.java

@@ -1,4 +1,4 @@
-package io.mkrzywanski.gpn.scrapper.domain.post;
+package io.mkrzywanski.gpn.scrapper.domain.post.price;
 
 import java.math.BigDecimal;
 import java.util.Currency;

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/FullGameInfo.java

@@ -0,0 +1,11 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+record FullGameInfo(Integer gameId, String gameName, GameInfo gameInfo) {
+    boolean isSuccess() {
+        return gameInfo().isSuccess();
+    }
+
+    boolean isOnSale() {
+        return gameInfo.isOnSale();
+    }
+}

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/GameData.java

@@ -0,0 +1,9 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+record GameData(@JsonProperty("price_overview") PriceOverview priceInfo) {
+    static GameData empty() {
+        return new GameData(new PriceOverview("PLN", -1, -1, -1));
+    }
+}

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/GameInfo.java

@@ -0,0 +1,40 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+record GameInfo(boolean success, GameData data) {
+    boolean isOnSale() {
+        return data.priceInfo().discount_percent() > 0;
+    }
+
+    boolean isSuccess() {
+        return this.success;
+    }
+
+    static class Builder {
+        private boolean success;
+        private GameData gameData = GameData.empty();
+
+        Builder withSuccessValue(final boolean success) {
+            this.success = success;
+            return this;
+        }
+
+        Builder withSuccess() {
+            this.success = true;
+            return this;
+        }
+
+        Builder withFailure() {
+            this.success = false;
+            return this;
+        }
+
+        Builder withGameData(final GameData gameData) {
+            this.gameData = gameData;
+            return this;
+        }
+
+        GameInfo build() {
+            return new GameInfo(success, gameData);
+        }
+    }
+}

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/GameInfoDeserializer.java

@@ -0,0 +1,49 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+
+import java.io.IOException;
+
+/**
+ * Since steam api sometimes returns json array in "data" field instead of object, this serializer is used
+ * to detect such cases and skip them as failures.
+ */
+class GameInfoDeserializer extends StdDeserializer<GameInfo> {
+    protected GameInfoDeserializer() {
+        super(GameInfo.class);
+    }
+
+    @Override
+    public GameInfo deserialize(final JsonParser parser, final DeserializationContext ctxt) throws IOException {
+
+        final var gameDataBuilder = new GameInfo.Builder();
+        while (!parser.isClosed()) {
+            final JsonToken jsonToken = parser.nextToken();
+
+            if (JsonToken.FIELD_NAME.equals(jsonToken)) {
+                final String fieldName = parser.getCurrentName();
+
+                parser.nextToken();
+
+                if ("success".equals(fieldName)) {
+                    gameDataBuilder.withSuccessValue(parser.getBooleanValue());
+                } else if ("data".equals(fieldName)) {
+                    handleDataFieldDeserialization(parser, gameDataBuilder);
+                }
+            }
+        }
+        return gameDataBuilder.build();
+    }
+
+    private void handleDataFieldDeserialization(final JsonParser parser, final GameInfo.Builder gameData) throws IOException {
+        final boolean isNotArrayStart = !JsonToken.START_ARRAY.equals(parser.currentToken());
+        if (isNotArrayStart) {
+            gameData.withGameData(parser.readValueAs(GameData.class));
+        } else {
+            gameData.withFailure();
+        }
+    }
+}

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/JsonMappingErrorHandlingDeserializer.java

@@ -0,0 +1,27 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+
+import java.io.IOException;
+
+class JsonMappingErrorHandlingDeserializer<T> extends JsonDeserializer<T> {
+    private final StdDeserializer<T> delegate;
+
+    JsonMappingErrorHandlingDeserializer(final StdDeserializer<T> delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public T deserialize(final JsonParser jp, final DeserializationContext ctxt) throws IOException {
+        try {
+            return delegate.deserialize(jp, ctxt);
+        } catch (final JsonMappingException e) {
+            // If a JSON Mapping occurs, simply returning null instead of blocking things
+            return null;
+        }
+    }
+}

promotion-notifier-backend/pn-site-scrapper/scrapper-domain/src/main/java/io/mkrzywanski/gpn/scrapper/domain/steam/PriceOverview.java

@@ -0,0 +1,6 @@
+package io.mkrzywanski.gpn.scrapper.domain.steam;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+record PriceOverview(String currency, int initial, int discount_percent, @JsonProperty("final") int finalPrice) {
+}