Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SSE 수정 #54

Merged
merged 3 commits into from
Nov 12, 2023
Merged

SSE 수정 #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ec98a0c
:sparkles: feat: create Stt data dto #53
noparamin Nov 12, 2023
151f0dc
:hammer: feat: change request feature SSE #53
noparamin Nov 12, 2023
2c3a832
:hammer: feat: add SSE for STT #53
noparamin Nov 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 2 additions & 61 deletions src/main/java/com/smart/watchboard/controller/AudioFileController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,37 +54,7 @@ public ResponseEntity<?> uploadAudioFile(@PathVariable(value = "documentID") lon
// s3에 오디오 파일 저장
S3Dto s3Dto = new S3Dto(audioFile, documentId, userId, "mp3");
String path = awsS3Uploader.uploadFile(s3Dto);
// STT
//String sttResult = sttService.getSTT(path);
ResponseEntity<String> sttResponseEntity = sttService.getSTT(path);
String sttResult = sttService.getText(sttResponseEntity);
// convert
String sttFileName = String.valueOf(userId) + "_" + String.valueOf(documentId) + ".pdf";
File textPdfFile = convertStringToPdf(sttResult, sttFileName);

String contentType = "application/pdf";
String originalFilename = textPdfFile.getName();
String name = textPdfFile.getName();

FileInputStream fileInputStream = new FileInputStream(textPdfFile);
MultipartFile multipartFile = new MockMultipartFile(name, originalFilename, contentType, fileInputStream);
S3Dto s3DtoForSTT = new S3Dto(multipartFile, documentId, userId, "pdf");
String textPdfPath = awsS3Uploader.uploadTextPdfFile(s3DtoForSTT);

List<SttData> data = sttService.getSTTData(sttResponseEntity);
lectureNoteService.createLectureNote(documentId, data, sttResult);

// ResponseEntity<KeywordsBodyDto> responseEntity = requestService.requestSTTKeywords(textPdfPath);
// List<String> keywords = keywordService.createKeywords(responseEntity, documentId);
//
// // 요약본 요청
// ResponseEntity<SummaryDto> summary = requestService.requestSTTSummary(textPdfPath);
// summaryService.createSummary(documentId, summary.getBody().getSummary());
sseService.notifyKeywords(documentId, textPdfPath);
sseService.notifySummary(documentId, textPdfPath);


//UploadDto uploadDto = new UploadDto(keywords, data);
sseService.notifySTT(documentId, path, userId);

whiteboardService.setDataType(documentId, "audio");

Expand All @@ -99,36 +69,7 @@ public ResponseEntity<?> updateAudioFile(@PathVariable(value = "documentID") lon

S3Dto s3Dto = new S3Dto(audioFile, documentId, userId, "mp3");
String path = awsS3Uploader.uploadFile(s3Dto);

ResponseEntity<String> sttResponseEntity = sttService.getSTT(path);
String sttResult = sttService.getText(sttResponseEntity);
List<SttData> data = sttService.getSTTData(sttResponseEntity);
lectureNoteService.updateLectureNote(documentId, data, sttResult);

// convert

String sttFileName = String.valueOf(userId) + "_" + String.valueOf(documentId) + ".pdf";
File textPdfFile = convertStringToPdf(sttResult, sttFileName);

String contentType = "application/pdf";
String originalFilename = textPdfFile.getName();
String name = textPdfFile.getName();

FileInputStream fileInputStream = new FileInputStream(textPdfFile);
MultipartFile multipartFile = new MockMultipartFile(name, originalFilename, contentType, fileInputStream);
S3Dto s3DtoForSTT = new S3Dto(multipartFile, documentId, userId, "pdf");
String textPdfPath = awsS3Uploader.uploadTextPdfFile(s3DtoForSTT);

sseService.notifyKeywords(documentId, textPdfPath);
sseService.notifySummary(documentId, textPdfPath);

// ResponseEntity<KeywordsBodyDto> responseEntity = requestService.requestSTTKeywords(textPdfPath);
// List<String> keywords = keywordService.renewKeywords(responseEntity, documentId); // update
//
// // 요약본 요청
// ResponseEntity<SummaryDto> summary = requestService.requestSTTSummary(sttResult);
// summaryService.updateSummary(documentId, summary.getBody().getSummary()); // update
// UploadDto uploadDto = new UploadDto(keywords, data);
sseService.notifySTT(documentId, path, userId);

whiteboardService.setDataType(documentId, "audio");

Expand Down
5 changes: 3 additions & 2 deletions src/main/java/com/smart/watchboard/controller/GraphController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,10 @@ public ResponseEntity<?> updateKeywords(@PathVariable(value = "documentID") long
@GetMapping("/documents/{documentID}/mindmap/keyword/{keywordLabel}")
@Operation(summary = "키워드 질문", description = "키워드 AI에 질문")
public ResponseEntity<?> getAnswer(@PathVariable(value = "documentID") long documentId, @PathVariable String keywordLabel, @RequestHeader("Authorization") String accessToken) throws JsonProcessingException {
ResponseEntity<AnswerDto> responseEntity = requestService.requestAnswer(documentId, keywordLabel);
//ResponseEntity<AnswerDto> responseEntity = requestService.requestAnswer(documentId, keywordLabel);
sseService.notifyAnswer(documentId, keywordLabel);

return new ResponseEntity<>(responseEntity, HttpStatus.OK);
return new ResponseEntity<>(HttpStatus.OK);
}

@GetMapping(value = "documents/{documentID}/subscribe", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/com/smart/watchboard/dto/SttDataDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.smart.watchboard.dto;

import com.smart.watchboard.domain.SttData;
import lombok.Getter;
import lombok.Setter;

import java.util.List;

@Getter
@Setter
public class SttDataDto {
private List<SttData> data;

public SttDataDto(List<SttData> data) {
this.data = data;
}
}
99 changes: 92 additions & 7 deletions src/main/java/com/smart/watchboard/service/SseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
package com.smart.watchboard.service;

import com.itextpdf.text.DocumentException;
import com.smart.watchboard.common.support.AwsS3Uploader;
import com.smart.watchboard.domain.Note;
import com.smart.watchboard.dto.AnswerDto;
import com.smart.watchboard.dto.KeywordsBodyDto;
import com.smart.watchboard.dto.MindmapResponseDto;
import com.smart.watchboard.dto.SummaryDto;
import com.smart.watchboard.domain.SttData;
import com.smart.watchboard.dto.*;
import com.smart.watchboard.repository.EmitterRepository;
import lombok.RequiredArgsConstructor;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;

import static com.smart.watchboard.common.support.PdfConverter.convertStringToPdf;

@Service
@RequiredArgsConstructor
public class SseService {
Expand All @@ -22,10 +28,12 @@ public class SseService {
private final FileService fileService;
private final WhiteboardService whiteboardService;
private final RequestService requestService;
//private final LectureNoteService lectureNoteService;
private final LectureNoteService lectureNoteService;
private final NoteService noteService;
private final KeywordService keywordService;
private final SummaryService summaryService;
private final STTService sttService;
private final AwsS3Uploader awsS3Uploader;

public SseEmitter subscribe(Long documentId) {
SseEmitter emitter = createEmitter(documentId);
Expand Down Expand Up @@ -78,8 +86,17 @@ public void notifySummary(Long documentId, String path) {
sendSummary(documentId, path);
}

public void notifyAnswer(Long documentId, String path) {
sendAnswer(documentId, path);
public void notifyAnswer(Long documentId, String keywordLabel) {
sendAnswer(documentId, keywordLabel);
}

public void notifySTT(Long documentId, String path, Long userId) {
Note note = noteService.findByDocument(documentId);
if (note == null) {
sendSTT(documentId, path, userId);
} else {
sendSTTUpdate(documentId, path, userId);
}
}

private void sendKeywords(Long documentId, String path) {
Expand Down Expand Up @@ -135,6 +152,74 @@ private void sendAnswer(Long documentId, String keywordLabel) {

}

private void sendSTT(Long documentId, String path, Long userId) {
SseEmitter emitter = emitterRepository.get(documentId);
if (emitter != null) {
try {
ResponseEntity<String> sttResponseEntity = sttService.getSTT(path);
String sttResult = sttService.getText(sttResponseEntity);

List<SttData> data = sttService.getSTTData(sttResponseEntity);
lectureNoteService.createLectureNote(documentId, data, sttResult);
SttDataDto sttDataDto = new SttDataDto(data);

String sttFileName = String.valueOf(userId) + "_" + String.valueOf(documentId) + ".pdf";
File textPdfFile = convertStringToPdf(sttResult, sttFileName);

String contentType = "application/pdf";
String originalFilename = textPdfFile.getName();
String name = textPdfFile.getName();

FileInputStream fileInputStream = new FileInputStream(textPdfFile);
MultipartFile multipartFile = new MockMultipartFile(name, originalFilename, contentType, fileInputStream);
S3Dto s3DtoForSTT = new S3Dto(multipartFile, documentId, userId, "pdf");
String textPdfPath = awsS3Uploader.uploadTextPdfFile(s3DtoForSTT);

notifyKeywords(documentId, textPdfPath);
notifySummary(documentId, textPdfPath);

emitter.send(SseEmitter.event().id(String.valueOf(documentId)).name("audio").data(sttDataDto));
} catch (IOException | DocumentException exception) {
emitterRepository.deleteById(documentId);
emitter.completeWithError(exception);
}
}
}

private void sendSTTUpdate(Long documentId, String path, Long userId) {
SseEmitter emitter = emitterRepository.get(documentId);
if (emitter != null) {
try {
ResponseEntity<String> sttResponseEntity = sttService.getSTT(path);
String sttResult = sttService.getText(sttResponseEntity);

List<SttData> data = sttService.getSTTData(sttResponseEntity);
lectureNoteService.updateLectureNote(documentId, data, sttResult);
SttDataDto sttDataDto = new SttDataDto(data);

String sttFileName = String.valueOf(userId) + "_" + String.valueOf(documentId) + ".pdf";
File textPdfFile = convertStringToPdf(sttResult, sttFileName);

String contentType = "application/pdf";
String originalFilename = textPdfFile.getName();
String name = textPdfFile.getName();

FileInputStream fileInputStream = new FileInputStream(textPdfFile);
MultipartFile multipartFile = new MockMultipartFile(name, originalFilename, contentType, fileInputStream);
S3Dto s3DtoForSTT = new S3Dto(multipartFile, documentId, userId, "pdf");
String textPdfPath = awsS3Uploader.uploadTextPdfFile(s3DtoForSTT);

notifyKeywords(documentId, textPdfPath);
notifySummary(documentId, textPdfPath);

emitter.send(SseEmitter.event().id(String.valueOf(documentId)).name("audio").data(sttDataDto));
} catch (IOException | DocumentException exception) {
emitterRepository.deleteById(documentId);
emitter.completeWithError(exception);
}
}
}

private SseEmitter createEmitter(Long documentId) {
SseEmitter emitter = new SseEmitter(DEFAULT_TIMEOUT);
emitterRepository.save(documentId, emitter);
Expand Down