Merge pull request #46 from kakao-tech-campus-2nd-step3/weekly/6

CICD 테스트

.github/workflows/gradle.yml

@@ -6,50 +6,50 @@ on:
     branches: [ "Master", "develop", "weekly/*" ]
 
 permissions:
-      contents: read
+  contents: read
 
 jobs:
   # Spring Boot 애플리케이션을 빌드하여 도커허브에 푸시하는 과정
   build-docker-image:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-    # Java 17 세팅
-    - name: Set up JDK 17
-      uses: actions/setup-java@v4
-      with:
-        java-version: '17'
-        distribution: 'temurin'
-
-    # Gradle Wrapper 파일 실행 권한 주기
-    - name: Run chmod to make gradlew executable
-      run: chmod +x ./gradlew
-
-    # Spring Boot 애플리케이션 빌드
-    - name: Build with Gradle
-      uses: gradle/gradle-build-action@67421db6bd0bf253fb4bd25b31ebb98943c375e1
-      with:
-        arguments: clean bootJar
-
-    # 테스트 실행
-#    - name: Run tests
-#      run: ./gradlew test
-
-    # Docker 이미지 빌드
-    - name: docker image build
-      run: docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/inplace .
-
-    # DockerHub 로그인
-    - name: docker login
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-    # Docker Hub 이미지 푸시
-    - name: docker Hub push
-      run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/inplace
+      - uses: actions/checkout@v4
+      # Java 17 세팅
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+
+      # Gradle Wrapper 파일 실행 권한 주기
+      - name: Run chmod to make gradlew executable
+        run: chmod +x ./gradlew
+
+      # Spring Boot 애플리케이션 빌드
+      - name: Build with Gradle
+        uses: gradle/gradle-build-action@67421db6bd0bf253fb4bd25b31ebb98943c375e1
+        with:
+          arguments: clean bootJar
+
+      # 테스트 실행
+      #    - name: Run tests
+      #      run: ./gradlew test
+
+      # Docker 이미지 빌드
+      - name: docker image build
+        run: docker build -t ${{ secrets.DOCKERHUB_USERNAME }}/inplace .
+
+      # DockerHub 로그인
+      - name: docker login
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      # Docker Hub 이미지 푸시
+      - name: docker Hub push
+        run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/inplace
 
 
   # 위 과정에서 푸시한 이미지를 ec2에서 풀받아서 실행시키는 과정 
@@ -73,12 +73,7 @@ jobs:
       # .env 파일 생성
       - name: Create .env file
         run: |
-          echo "KAKAO_CLIENT_ID=${{ secrets.KAKAO_CLIENT_ID }}" >> .env
-          echo "KAKAO_CLIENT_SECRET=${{ secrets.KAKAO_CLIENT_SECRET }}" >> .env
-          echo "KAKAO_REDIRECT_URI=${{ secrets.KAKAO_REDIRECT_URI }}" >> .env
-          echo "JWT_SECRET=${{ secrets.JWT_SECRET }}" >> .env
-          echo "JWT_ACCESS_TOKEN_EXPIRED_TIME=${{ secrets.JWT_ACCESS_TOKEN_EXPIRED_TIME }}" >> .env
-          echo "JWT_REFRESH_TOKEN_EXPIRED_TIME=${{ secrets.JWT_REFRESH_TOKEN_EXPIRED_TIME }}" >> .env
+          echo "${{ secrets.ENV }}" >> .env
 
       # 브랜치에 따라 다른 환경 변수를 설정하여 컨테이너 실행
       - name: docker run new container
@@ -88,7 +83,7 @@ jobs:
           elif [ "${{ github.ref }}" == "refs/heads/develop" ]; then
             sudo docker run --name inplace-dev --rm -d -p 8081:8080 --env-file .env ${{ secrets.DOCKERHUB_USERNAME }}/inplace
           fi
-          
+
       # 미사용 이미지를 정리
       - name: delete old docker image
         run: sudo docker system prune -f

.gitignore

@@ -205,7 +205,9 @@ gradle-app.setting
 .env
 /src/test/resources/application-test.yaml
 
-# CommandLineRunner for loading dummy data
-src/main/java/team7/inplace/place/config/DataLoader.java
+# Ignore specific application config files
+application.yaml
+application-security.yaml
+application-youtube.yaml
 
 # End of https://www.toptal.com/developers/gitignore/api/macos,java,intellij,gradle

build.gradle

@@ -36,6 +36,7 @@ dependencies {
     implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.0.3'
 
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
+    testImplementation 'org.springframework.security:spring-security-test'
     runtimeOnly 'com.mysql:mysql-connector-j'
     runtimeOnly 'com.h2database:h2'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'

src/main/java/team7/inplace/crawling/application/AddressUtil.java

@@ -0,0 +1,25 @@
+package team7.inplace.crawling.application;
+
+import static lombok.AccessLevel.PRIVATE;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor(access = PRIVATE)
+public class AddressUtil {
+    private static final String ADDRESS_REGEX = "[가-힣0-9]+(?:도|시|구|군|읍|면|동|리|로|길)[^#,\\n()]+(?:동|읍|면|리|로|길|호|층|번지)[^#,\\n()]+";
+
+    public static String extractAddress(JsonNode snippet) {
+
+        String videoDescription = snippet.path("description").asText();
+
+        Pattern pattern = Pattern.compile(ADDRESS_REGEX);
+        Matcher matcher = pattern.matcher(videoDescription);
+        if (matcher.find()) {
+            return matcher.group();
+        }
+        return null;
+    }
+}

src/main/java/team7/inplace/crawling/application/CrawlingFacade.java

@@ -0,0 +1,22 @@
+package team7.inplace.crawling.application;
+
+import lombok.RequiredArgsConstructor;
+import team7.inplace.global.annotation.Facade;
+import team7.inplace.video.application.VideoFacade;
+
+@Facade
+@RequiredArgsConstructor
+public class CrawlingFacade {
+    private final YoutubeCrawlingService youtubeCrawlingService;
+    private final VideoFacade videoFacade;
+
+    public void updateVideos() {
+        var crawlingInfos = youtubeCrawlingService.crawlAllVideos();
+        for (var crawlingInfo : crawlingInfos) {
+            var videoCommands = crawlingInfo.toVideoCommands();
+            var placesCommands = crawlingInfo.toPlacesCommands();
+
+            videoFacade.createVideos(videoCommands, placesCommands);
+        }
+    }
+}

src/main/java/team7/inplace/crawling/application/YoutubeCrawlingService.java

@@ -1,9 +1,11 @@
 package team7.inplace.crawling.application;
 
+import java.util.List;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
+import team7.inplace.crawling.application.dto.CrawlingInfo;
 import team7.inplace.crawling.client.KakaoMapClient;
 import team7.inplace.crawling.client.YoutubeClient;
 import team7.inplace.crawling.persistence.YoutubeChannelRepository;
@@ -22,17 +24,30 @@ public class YoutubeCrawlingService {
         3. 마지막 비디오 UUID를 업데이트 한다.
         4. 카카오 API를 호출해 장소 정보를 가져온다
      */
-    public void crawlAllVideos() {
+    public List<CrawlingInfo> crawlAllVideos() {
         var youtubeChannels = youtubeChannelRepository.findAll();
-        for (var channel : youtubeChannels) {
-            var rawVideoInfos = youtubeClient.getVideos(channel.getPlayListUUID(), channel.getLastVideoUUID());
-            channel.updateLastVideoUUID(rawVideoInfos.get(0).videoId());
-
-            var videos = rawVideoInfos.stream()
-                    .map(rawVideoInfo -> kakaoMapClient.search(rawVideoInfo, channel.getChannelType().getCode()))
-                    .filter(Objects::nonNull)
-                    .toList();
-        }
+
+        var crawlInfos = youtubeChannels.stream()
+                .map(channel -> {
+                    var videoSnippets = youtubeClient.getVideos(channel.getPlayListUUID(), channel.getLastVideoUUID());
+
+                    var videoAddresses = videoSnippets.stream()
+                            .map(AddressUtil::extractAddress)
+                            .toList();
+
+                    var placeNodes = videoAddresses.stream()
+                            .map(address -> {
+                                if (Objects.isNull(address)) {
+                                    return null;
+                                }
+                                return kakaoMapClient.search(address, channel.getChannelType().getCode());
+                            })
+                            .toList();
+
+                    return new CrawlingInfo(channel.getInfluencerId(), videoSnippets, placeNodes);
+                }).toList();
+
+        return crawlInfos;
     }
 }
 

src/main/java/team7/inplace/crawling/application/dto/CrawlingInfo.java

@@ -0,0 +1,25 @@
+package team7.inplace.crawling.application.dto;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.List;
+import team7.inplace.crawling.client.dto.PlaceNode;
+import team7.inplace.place.application.command.PlacesCommand;
+import team7.inplace.video.application.command.VideoCommand;
+
+public record CrawlingInfo(
+        Long influencerId,
+        List<JsonNode> videoSnippets,
+        List<PlaceNode> placeNodes
+) {
+    public List<VideoCommand.Create> toVideoCommands() {
+        return videoSnippets.stream()
+                .map(snippet -> VideoCommand.Create.from(snippet, influencerId))
+                .toList();
+    }
+
+    public List<PlacesCommand.Create> toPlacesCommands() {
+        return placeNodes.stream()
+                .map(placeNode -> PlacesCommand.Create.from(placeNode.locationNode(), placeNode.placeNode()))
+                .toList();
+    }
+}

src/main/java/team7/inplace/crawling/client/KakaoMapClient.java

@@ -10,8 +10,7 @@
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
-import team7.inplace.crawling.client.dto.RawPlace;
-import team7.inplace.crawling.client.dto.RawVideoInfo;
+import team7.inplace.crawling.client.dto.PlaceNode;
 import team7.inplace.global.kakao.config.KakaoApiProperties;
 
 @Slf4j
@@ -24,8 +23,7 @@ public class KakaoMapClient {
     private final KakaoApiProperties kakaoApiProperties;
     private final RestTemplate restTemplate;
 
-    public RawPlace.Info search(RawVideoInfo videoInfo, String category) {
-        var address = videoInfo.address();
+    public PlaceNode search(String address, String category) {
         var locationInfo = getLocateInfo(address, category);
         var placeId = locationInfo.has("documents") ?
                 locationInfo.get("documents").get(0).get("id").asText() : null;
@@ -34,7 +32,7 @@ public RawPlace.Info search(RawVideoInfo videoInfo, String category) {
         }
 
         var placeInfo = getPlaceInfo(placeId);
-        return RawPlace.Info.from(locationInfo, placeInfo);
+        return PlaceNode.of(locationInfo, placeInfo);
     }
 
     private JsonNode getLocateInfo(String address, String category) {

src/main/java/team7/inplace/crawling/client/YoutubeClient.java

@@ -4,31 +4,26 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
-import team7.inplace.crawling.client.dto.RawVideoInfo;
 
 @Slf4j
 @Component
 public class YoutubeClient {
     private static final String PLAY_LIST_ITEMS_BASE_URL = "https://www.googleapis.com/youtube/v3/playlistItems";
     private static final String PLAY_LIST_PARAMS = "?part=snippet&playlistId=%s&key=%s&maxResults=50";
-    private static final String ADDRESS_REGEX = "[가-힣0-9]+(?:도|시|구|군|읍|면|동|리|로|길)[^#,\\n()]+(?:동|읍|면|리|로|길|호|층|번지)[^#,\\n()]+";
     private final RestTemplate restTemplate;
     private final String apiKey;
 
     public YoutubeClient(@Value("${youtube.api.key}") String apiKey, RestTemplate restTemplate) {
-        log.info("Youtube API Key: {}", apiKey);
         this.restTemplate = restTemplate;
         this.apiKey = apiKey;
     }
 
-    public List<RawVideoInfo> getVideos(String playListId, String finalVideoUUID) {
-        List<RawVideoInfo> videoInfos = new ArrayList<>();
+    public List<JsonNode> getVideos(String playListId, String finalVideoUUID) {
+        List<JsonNode> snippets = new ArrayList<>();
         String nextPageToken = null;
         while (true) {
             String url = PLAY_LIST_ITEMS_BASE_URL + String.format(PLAY_LIST_PARAMS, playListId, apiKey);
@@ -41,15 +36,14 @@ public List<RawVideoInfo> getVideos(String playListId, String finalVideoUUID) {
                 response = restTemplate.getForObject(url, JsonNode.class);
             } catch (Exception e) {
                 log.error("Youtube API 호출이 실패했습니다. Youtuber Id {}", playListId);
-                log.info(e.getMessage());
                 break;
             }
             if (Objects.isNull(response)) {
                 log.error("Youtube API Response가 NULL입니다 {}.", playListId);
                 break;
             }
 
-            var containsLastVideo = extractRawVideoInfo(videoInfos, response.path("items"), finalVideoUUID);
+            var containsLastVideo = extractSnippets(snippets, response.path("items"), finalVideoUUID);
             if (containsLastVideo) {
                 break;
             }
@@ -58,39 +52,22 @@ public List<RawVideoInfo> getVideos(String playListId, String finalVideoUUID) {
                 break;
             }
         }
-        return videoInfos;
+        return snippets;
     }
 
     private boolean isLastPage(String nextPageToken) {
         return Objects.isNull(nextPageToken) || nextPageToken.isEmpty();
     }
 
-    private boolean extractRawVideoInfo(List<RawVideoInfo> videoInfos, JsonNode items, String finalVideoUUID) {
+    private boolean extractSnippets(List<JsonNode> snippets, JsonNode items, String finalVideoUUID) {
         for (JsonNode item : items) {
             var snippet = item.path("snippet");
             var videoId = snippet.path("resourceId").path("videoId").asText();
-            var videoTitle = snippet.path("title").asText();
-            var videoDescription = snippet.path("description").asText();
             if (videoId.equals(finalVideoUUID)) {
                 return true;
             }
-
-            var address = extractAddress(videoDescription);
-            if (Objects.nonNull(address)) {
-                videoInfos.add(new RawVideoInfo(videoId, videoTitle, address));
-                continue;
-            }
-            log.info("주소를 찾을 수 없습니다. {}", videoDescription);
+            snippets.add(snippet);
         }
         return false;
     }
-
-    private String extractAddress(String description) {
-        Pattern pattern = Pattern.compile(ADDRESS_REGEX);
-        Matcher matcher = pattern.matcher(description);
-        if (matcher.find()) {
-            return matcher.group();
-        }
-        return null;
-    }
 }

src/main/java/team7/inplace/crawling/client/dto/PlaceNode.java

@@ -0,0 +1,12 @@
+package team7.inplace.crawling.client.dto;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+public record PlaceNode(
+        JsonNode locationNode,
+        JsonNode placeNode
+) {
+    public static PlaceNode of(JsonNode locationNode, JsonNode placeNode) {
+        return new PlaceNode(locationNode, placeNode);
+    }
+}