From 7aa37d2e11d2d77600dcf406eeb23fc74b60ce61 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Fri, 28 Jul 2023 20:28:47 +0330 Subject: [PATCH 01/36] generate ebook get dstPath --- internal/core/ebook.go | 20 +++++++------------- internal/core/processing.go | 7 ++++--- internal/webserver/handler-api.go | 4 +++- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 522094185..8ba8c322d 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -16,7 +16,7 @@ import ( "github.com/pkg/errors" ) -func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { +func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) { // variable for store generated html code var html string @@ -40,34 +40,28 @@ func GenerateEbook(req ProcessRequest) (book model.Bookmark, err error) { if _, err := os.Stat(archivePath); err == nil { book.HasArchive = true } - ebookfile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) - // if epub exist finish prosess else continue - if _, err := os.Stat(ebookfile); err == nil { - book.HasEbook = true - return book, nil - } + contentType := req.ContentType if strings.Contains(contentType, "application/pdf") { return book, errors.New("can't create ebook for pdf") } - ebookDir := fp.Join(req.DataDir, "ebook") // check if directory not exsist create that - if _, err := os.Stat(ebookDir); os.IsNotExist(err) { - err := os.MkdirAll(ebookDir, model.DataDirPerm) + if _, err := os.Stat(dstPath); os.IsNotExist(err) { + err := os.MkdirAll(dstPath, model.DataDirPerm) if err != nil { return book, errors.Wrap(err, "can't create ebook directory") } } // create epub file - epubFile, err := os.Create(ebookfile) + dstFile, err := os.Create(fp.Join(dstPath, fmt.Sprintf("%d.epub", book.ID))) if err != nil { return book, errors.Wrap(err, "can't create ebook") } - defer epubFile.Close() + defer dstFile.Close() // Create zip archive - epubWriter := zip.NewWriter(epubFile) + epubWriter := zip.NewWriter(dstFile) defer epubWriter.Close() // Create the mimetype file diff --git a/internal/core/processing.go b/internal/core/processing.go index 12a2c1050..2b90605b0 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -128,13 +128,14 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create ebook as well if book.CreateEbook { - ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) - os.Remove(ebookPath) + // ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) + ebookPath := fp.Join(req.DataDir, "ebook") + // os.Remove(ebookPath) if strings.Contains(contentType, "application/pdf") { return book, false, errors.Wrap(err, "can't create ebook from pdf") } else { - _, err = GenerateEbook(req) + _, err = GenerateEbook(req, ebookPath) if err != nil { return book, true, errors.Wrap(err, "failed to create ebook") } diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index 8362a835f..c84a874f1 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -434,7 +434,9 @@ func (h *Handler) ApiDownloadEbook(w http.ResponseWriter, r *http.Request, ps ht ContentType: contentType, } - book, err = core.GenerateEbook(request) + //TODO: if file exist book return avilable file + ebookPath := fp.Join(request.DataDir, "ebook") + book, err = core.GenerateEbook(request, ebookPath) content.Close() if err != nil { From 91937b3e252eb6872a6cd588ff86e728a11b0ce2 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Fri, 28 Jul 2023 21:39:57 +0330 Subject: [PATCH 02/36] Archive and ebook can recover if download faild --- internal/core/processing.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index 2b90605b0..c148c6410 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -128,17 +128,20 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create ebook as well if book.CreateEbook { - // ebookPath := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) - ebookPath := fp.Join(req.DataDir, "ebook") - // os.Remove(ebookPath) + ebookFile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) + tmpebookfile := fp.Join(req.DataDir, "tmp/ebook", fmt.Sprintf("%d.epub", book.ID)) + tmpEbookPath := fp.Join(req.DataDir, "tmp/ebook") if strings.Contains(contentType, "application/pdf") { return book, false, errors.Wrap(err, "can't create ebook from pdf") } else { - _, err = GenerateEbook(req, ebookPath) + _, err = GenerateEbook(req, tmpEbookPath) if err != nil { + os.Remove(tmpebookfile) return book, true, errors.Wrap(err, "failed to create ebook") } + os.Remove(ebookFile) + os.Rename(tmpebookfile, ebookFile) book.HasEbook = true } } @@ -146,7 +149,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create offline archive as well if book.CreateArchive { archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - os.Remove(archivePath) + tmpArchivePath := fp.Join(req.DataDir, "tmp/archive", fmt.Sprintf("%d", book.ID)) archivalRequest := warc.ArchivalRequest{ URL: book.URL, @@ -156,11 +159,13 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, LogEnabled: req.LogArchival, } - err = warc.NewArchive(archivalRequest, archivePath) + err = warc.NewArchive(archivalRequest, tmpArchivePath) if err != nil { + os.Remove(tmpArchivePath) return book, false, fmt.Errorf("failed to create archive: %v", err) } - + os.Remove(archivePath) + os.Rename(tmpArchivePath, archivePath) book.HasArchive = true } From c12e3f213c0162abe2cd8b3748b681c233880816 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 11:57:33 +0330 Subject: [PATCH 03/36] recover thumb if download faild --- internal/core/processing.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index c148c6410..500ddae64 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -72,7 +72,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, nurl, err := url.Parse(book.URL) if err != nil { - return book, true, fmt.Errorf("Failed to parse url: %v", err) + return book, true, fmt.Errorf("failed to parse url: %v", err) } article, err := readability.FromReader(readabilityInput, nurl) @@ -117,13 +117,17 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Save article image to local disk strID := strconv.Itoa(book.ID) imgPath := fp.Join(req.DataDir, "thumb", strID) + tmpImgPath := fp.Join(req.DataDir, "tmp/thumb", strID) for _, imageURL := range imageURLs { - err = downloadBookImage(imageURL, imgPath) + err = downloadBookImage(imageURL, tmpImgPath) if err == nil { book.ImageURL = path.Join("/", "bookmark", strID, "thumb") + os.Remove(tmpImgPath) break } + os.Remove(imgPath) + os.Rename(tmpImgPath, imgPath) } // If needed, create ebook as well From 379a3ba476189863cc932f0adb6d629f3ba4a387 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 17:09:51 +0330 Subject: [PATCH 04/36] thumb image create just if image processing is sucssesful --- internal/core/processing.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index 500ddae64..ab734bbd1 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -117,17 +117,13 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Save article image to local disk strID := strconv.Itoa(book.ID) imgPath := fp.Join(req.DataDir, "thumb", strID) - tmpImgPath := fp.Join(req.DataDir, "tmp/thumb", strID) for _, imageURL := range imageURLs { - err = downloadBookImage(imageURL, tmpImgPath) + err = downloadBookImage(imageURL, imgPath) if err == nil { book.ImageURL = path.Join("/", "bookmark", strID, "thumb") - os.Remove(tmpImgPath) break } - os.Remove(imgPath) - os.Rename(tmpImgPath, imgPath) } // If needed, create ebook as well @@ -200,6 +196,11 @@ func downloadBookImage(url, dstPath string) error { if err != nil { return fmt.Errorf("failed to create image dir: %v", err) } + tmpFile, err := os.CreateTemp("", "image") + if err != nil { + return fmt.Errorf("failed to create temporary image file: %v", err) + } + defer os.Remove(tmpFile.Name()) dstFile, err := os.Create(dstPath) if err != nil { @@ -221,7 +222,7 @@ func downloadBookImage(url, dstPath string) error { imgRatio := float64(imgWidth) / float64(imgHeight) if imgWidth >= 600 && imgHeight >= 400 && imgRatio > 1.3 { - err = jpeg.Encode(dstFile, img, nil) + err = jpeg.Encode(tmpFile, img, nil) } else { // Create background bg := image.NewNRGBA(imgRect) @@ -246,12 +247,23 @@ func downloadBookImage(url, dstPath string) error { draw.Draw(bg, bgRect, fg, fgPosition, draw.Over) // Save to file - err = jpeg.Encode(dstFile, bg, nil) + err = jpeg.Encode(tmpFile, bg, nil) } if err != nil { return fmt.Errorf("failed to save image %s: %v", url, err) } + // Copy temporary file to destination + _, err = tmpFile.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("failed to rewind temporary image file: %v", err) + } + + _, err = io.Copy(dstFile, tmpFile) + if err != nil { + return fmt.Errorf("failed to copy image to the destination") + } + return nil } From 42afd233b4cca947020b7f4cd936180cca7869f6 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 18:57:08 +0330 Subject: [PATCH 05/36] create epub in tmp if it sucssesful copy to destination --- internal/core/ebook.go | 55 +++++++++++++++++++++++++++++-------- internal/core/processing.go | 32 ++++++++++----------- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 8ba8c322d..eba3fc6b4 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -46,22 +46,16 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err return book, errors.New("can't create ebook for pdf") } - // check if directory not exsist create that - if _, err := os.Stat(dstPath); os.IsNotExist(err) { - err := os.MkdirAll(dstPath, model.DataDirPerm) - if err != nil { - return book, errors.Wrap(err, "can't create ebook directory") - } - } - // create epub file - dstFile, err := os.Create(fp.Join(dstPath, fmt.Sprintf("%d.epub", book.ID))) + // create temporary epub file + tmpFile, err := os.CreateTemp("", "ebook*.epub") if err != nil { - return book, errors.Wrap(err, "can't create ebook") + return book, errors.Wrap(err, "can't create temporary EPUB file") } - defer dstFile.Close() + defer tmpFile.Close() + defer os.Remove(tmpFile.Name()) // Create zip archive - epubWriter := zip.NewWriter(dstFile) + epubWriter := zip.NewWriter(tmpFile) defer epubWriter.Close() // Create the mimetype file @@ -217,6 +211,43 @@ img { if err != nil { return book, errors.Wrap(err, "can't write into content.html") } + // close epub and tmpFile + err = epubWriter.Close() + if err != nil { + return book, errors.Wrap(err, "failed to close EPUB writer") + } + err = tmpFile.Close() + if err != nil { + return book, errors.Wrap(err, "failed to close temporary EPUB file") + } + // open temporary file again + tmpFile, err = os.Open(tmpFile.Name()) + if err != nil { + return book, errors.Wrap(err, "can't open temporary EPUB file") + } + defer tmpFile.Close() + + // create ebook directory if it need + err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) + + // create dstFile + dstFile, err := os.Create(dstPath + ".epub") + if err != nil { + return book, errors.Wrap(err, "can't create ebook in dstPath") + } + defer dstFile.Close() + + // Copy the content from the temporary file to the destination file + _, err = tmpFile.Seek(0, io.SeekStart) + if err != nil { + return book, errors.Wrap(err, "failed to rewind temporary ebook file") + } + + _, err = io.Copy(dstFile, tmpFile) + if err != nil { + return book, errors.Wrap(err, "failed to copy image to the destination") + } + book.HasEbook = true return book, nil } diff --git a/internal/core/processing.go b/internal/core/processing.go index ab734bbd1..5b7da0a82 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -128,20 +128,15 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create ebook as well if book.CreateEbook { - ebookFile := fp.Join(req.DataDir, "ebook", fmt.Sprintf("%d.epub", book.ID)) - tmpebookfile := fp.Join(req.DataDir, "tmp/ebook", fmt.Sprintf("%d.epub", book.ID)) - tmpEbookPath := fp.Join(req.DataDir, "tmp/ebook") + ebookPath := fp.Join(req.DataDir, "ebook", strID) if strings.Contains(contentType, "application/pdf") { return book, false, errors.Wrap(err, "can't create ebook from pdf") } else { - _, err = GenerateEbook(req, tmpEbookPath) + _, err = GenerateEbook(req, ebookPath) if err != nil { - os.Remove(tmpebookfile) return book, true, errors.Wrap(err, "failed to create ebook") } - os.Remove(ebookFile) - os.Rename(tmpebookfile, ebookFile) book.HasEbook = true } } @@ -191,23 +186,13 @@ func downloadBookImage(url, dstPath string) error { } // At this point, the download has finished successfully. - // Prepare destination file. - err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) - if err != nil { - return fmt.Errorf("failed to create image dir: %v", err) - } + // Create tmpFile tmpFile, err := os.CreateTemp("", "image") if err != nil { return fmt.Errorf("failed to create temporary image file: %v", err) } defer os.Remove(tmpFile.Name()) - dstFile, err := os.Create(dstPath) - if err != nil { - return fmt.Errorf("failed to create image file: %v", err) - } - defer dstFile.Close() - // Parse image and process it. // If image is smaller than 600x400 or its ratio is less than 4:3, resize. // Else, save it as it is. @@ -254,6 +239,17 @@ func downloadBookImage(url, dstPath string) error { return fmt.Errorf("failed to save image %s: %v", url, err) } + // Prepare destination file. + err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) + if err != nil { + return fmt.Errorf("failed to create image dir: %v", err) + } + + dstFile, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("failed to create image file: %v", err) + } + defer dstFile.Close() // Copy temporary file to destination _, err = tmpFile.Seek(0, io.SeekStart) if err != nil { From 624a74638a00e38b0f82e3432988358f3b4bdf49 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 20:03:45 +0330 Subject: [PATCH 06/36] archive file create in tmp if it successful move to destination --- internal/core/processing.go | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index 5b7da0a82..c7e59ced3 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -143,8 +143,11 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create offline archive as well if book.CreateArchive { - archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - tmpArchivePath := fp.Join(req.DataDir, "tmp/archive", fmt.Sprintf("%d", book.ID)) + tmpFile, err := os.CreateTemp("", "archive") + if err != nil { + return book, false, fmt.Errorf("failed to create temp archive: %v", err) + } + defer os.Remove(tmpFile.Name()) archivalRequest := warc.ArchivalRequest{ URL: book.URL, @@ -154,13 +157,34 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, LogEnabled: req.LogArchival, } - err = warc.NewArchive(archivalRequest, tmpArchivePath) + err = warc.NewArchive(archivalRequest, tmpFile.Name()) if err != nil { - os.Remove(tmpArchivePath) + defer os.Remove(tmpFile.Name()) return book, false, fmt.Errorf("failed to create archive: %v", err) } - os.Remove(archivePath) - os.Rename(tmpArchivePath, archivePath) + + // Prepare destination file. + archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) + err = os.MkdirAll(fp.Dir(archivePath), model.DataDirPerm) + if err != nil { + return book, false, fmt.Errorf("failed to create destination directory archive: %v", err) + } + dstFile, err := os.Create(archivePath) + if err != nil { + return book, false, fmt.Errorf("failed to create destination archive: %v", err) + } + defer dstFile.Close() + // Copy temporary file to destination + _, err = tmpFile.Seek(0, io.SeekStart) + if err != nil { + return book, false, fmt.Errorf("failed to rewind temporary archive file: %v", err) + } + + _, err = io.Copy(dstFile, tmpFile) + if err != nil { + return book, false, fmt.Errorf("failed to copy archive to the destination %v", err) + } + book.HasArchive = true } From 2ec8e5ebdcd468f324e112a7630d4489907f7813 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 20:46:08 +0330 Subject: [PATCH 07/36] move to destination as function --- internal/core/ebook.go | 20 ++----------------- internal/core/processing.go | 39 ++++++++++++++++--------------------- 2 files changed, 19 insertions(+), 40 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index eba3fc6b4..7f4281daf 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -227,25 +227,9 @@ img { } defer tmpFile.Close() - // create ebook directory if it need - err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) - - // create dstFile - dstFile, err := os.Create(dstPath + ".epub") - if err != nil { - return book, errors.Wrap(err, "can't create ebook in dstPath") - } - defer dstFile.Close() - - // Copy the content from the temporary file to the destination file - _, err = tmpFile.Seek(0, io.SeekStart) - if err != nil { - return book, errors.Wrap(err, "failed to rewind temporary ebook file") - } - - _, err = io.Copy(dstFile, tmpFile) + err = MoveToDestination(dstPath, tmpFile) if err != nil { - return book, errors.Wrap(err, "failed to copy image to the destination") + return book, errors.Wrap(err, "failed move ebook to destination") } book.HasEbook = true diff --git a/internal/core/processing.go b/internal/core/processing.go index c7e59ced3..1da0654f3 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -164,25 +164,11 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, } // Prepare destination file. - archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - err = os.MkdirAll(fp.Dir(archivePath), model.DataDirPerm) - if err != nil { - return book, false, fmt.Errorf("failed to create destination directory archive: %v", err) - } - dstFile, err := os.Create(archivePath) - if err != nil { - return book, false, fmt.Errorf("failed to create destination archive: %v", err) - } - defer dstFile.Close() - // Copy temporary file to destination - _, err = tmpFile.Seek(0, io.SeekStart) - if err != nil { - return book, false, fmt.Errorf("failed to rewind temporary archive file: %v", err) - } + dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - _, err = io.Copy(dstFile, tmpFile) + err = MoveToDestination(dstPath, tmpFile) if err != nil { - return book, false, fmt.Errorf("failed to copy archive to the destination %v", err) + return book, false, fmt.Errorf("failed move archive to destination `: %v", err) } book.HasArchive = true @@ -263,26 +249,35 @@ func downloadBookImage(url, dstPath string) error { return fmt.Errorf("failed to save image %s: %v", url, err) } + err = MoveToDestination(dstPath, tmpFile) + if err != nil { + return err + } + + return nil +} + +func MoveToDestination(dstPath string, tmpFile *os.File) error { // Prepare destination file. - err = os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) + err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) if err != nil { - return fmt.Errorf("failed to create image dir: %v", err) + return fmt.Errorf("failed to create destination dir: %v", err) } dstFile, err := os.Create(dstPath) if err != nil { - return fmt.Errorf("failed to create image file: %v", err) + return fmt.Errorf("failed to create destination file: %v", err) } defer dstFile.Close() // Copy temporary file to destination _, err = tmpFile.Seek(0, io.SeekStart) if err != nil { - return fmt.Errorf("failed to rewind temporary image file: %v", err) + return fmt.Errorf("failed to rewind temporary file: %v", err) } _, err = io.Copy(dstFile, tmpFile) if err != nil { - return fmt.Errorf("failed to copy image to the destination") + return fmt.Errorf("failed to copy file to the destination") } return nil From 44b9a13c048f15aebf56f351e045b25137c6b488 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sat, 5 Aug 2023 21:33:12 +0330 Subject: [PATCH 08/36] update ebook download api and remove .epub from file name --- internal/webserver/handler-api.go | 36 +++++++++++++++++++++++-------- internal/webserver/handler-ui.go | 6 +++--- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index c84a874f1..cbbeb81b0 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -110,7 +110,7 @@ func (h *Handler) ApiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt strID := strconv.Itoa(bookmarks[i].ID) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + ebookPath := fp.Join(h.DataDir, "ebook", strID) if fileExists(imgPath) { bookmarks[i].ImageURL = path.Join(h.RootPath, "bookmark", strID, "thumb") @@ -285,7 +285,7 @@ func (h *Handler) ApiDeleteBookmark(w http.ResponseWriter, r *http.Request, ps h strID := strconv.Itoa(id) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + ebookPath := fp.Join(h.DataDir, "ebook", strID) os.Remove(imgPath) os.Remove(archivePath) @@ -434,14 +434,32 @@ func (h *Handler) ApiDownloadEbook(w http.ResponseWriter, r *http.Request, ps ht ContentType: contentType, } - //TODO: if file exist book return avilable file - ebookPath := fp.Join(request.DataDir, "ebook") - book, err = core.GenerateEbook(request, ebookPath) - content.Close() + // if file exist book return avilable file + strID := strconv.Itoa(book.ID) + ebookPath := fp.Join(request.DataDir, "ebook", strID) + _, err = os.Stat(ebookPath) + if err == nil { + // file already exists, return the existing file + imagePath := fp.Join(request.DataDir, "thumb", fmt.Sprintf("%d", book.ID)) + archivePath := fp.Join(request.DataDir, "archive", fmt.Sprintf("%d", book.ID)) + + if _, err := os.Stat(imagePath); err == nil { + book.ImageURL = fp.Join("/", "bookmark", strID, "thumb") + } - if err != nil { - chProblem <- book.ID - return + if _, err := os.Stat(archivePath); err == nil { + book.HasArchive = true + } + book.HasEbook = true + } else { + // generate ebook file + book, err = core.GenerateEbook(request, ebookPath) + content.Close() + + if err != nil { + chProblem <- book.ID + return + } } // Update list of bookmarks diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go index 2f300aea4..17800d169 100644 --- a/internal/webserver/handler-ui.go +++ b/internal/webserver/handler-ui.go @@ -48,8 +48,8 @@ func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, p } } - // Check if it has archive. - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + // Check if it has ebook. + ebookPath := fp.Join(h.DataDir, "ebook", strID) if fileExists(ebookPath) { bookmark.HasEbook = true } @@ -320,7 +320,7 @@ func (h *Handler) ServeBookmarkEbook(w http.ResponseWriter, r *http.Request, ps } // Check if it has ebook. - ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") + ebookPath := fp.Join(h.DataDir, "ebook", strID) if !fileExists(ebookPath) { http.Error(w, "ebook not found", http.StatusNotFound) return From 622ff9d275cfe8730127df3a152da26a189ada37 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 00:28:10 +0330 Subject: [PATCH 09/36] report faild item to user --- internal/view/assets/js/page/home.js | 38 +++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/internal/view/assets/js/page/home.js b/internal/view/assets/js/page/home.js index 1ff6feddd..9cf5f63fa 100644 --- a/internal/view/assets/js/page/home.js +++ b/internal/view/assets/js/page/home.js @@ -702,15 +702,35 @@ export default { this.editMode = false; this.dialog.loading = false; this.dialog.visible = false; - - json.forEach(book => { - var item = items.find(el => el.id === book.id); - this.bookmarks.splice(item.index, 1, book); - }); - }).catch(err => { - this.selection = []; - this.editMode = false; - this.dialog.loading = false; + + let faildUpdateArchives = []; + let faildCreateEbook = []; + json.forEach(book => { + var item = items.find(el => el.id === book.id); + this.bookmarks.splice(item.index, 1, book); + + if (data.createArchive && !book.hasArchive){ + faildUpdateArchives.push(book.id); + } + if (data.createEbook && !book.hasEbook){ + faildCreateEbook.push(book.id); + } + }), + + this.showDialog({ + title: `Update Archive Error`, + content: `Bookmarks Update Archive Faild : ${faildUpdateArchives.join(", ")} + Bookmarks Create Ebook Faild: ${faildCreateEbook.join(",")} + We recovered the last available version.`, + mainText: "OK", + mainClick: () => { + this.dialog.visible = false; + }, + }) + }).catch(err => { + this.selection = []; + this.editMode = false; + this.dialog.loading = false; this.getErrorMessage(err).then(msg => { this.showErrorDialog(msg); From bf31c3b4284a9caa23802e2337a142cda9015d7f Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 01:24:49 +0330 Subject: [PATCH 10/36] not show dialog if error not happen --- internal/view/assets/js/page/home.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/view/assets/js/page/home.js b/internal/view/assets/js/page/home.js index 9cf5f63fa..347e55f4f 100644 --- a/internal/view/assets/js/page/home.js +++ b/internal/view/assets/js/page/home.js @@ -715,8 +715,8 @@ export default { if (data.createEbook && !book.hasEbook){ faildCreateEbook.push(book.id); } - }), - + }); + if(faildCreateEbook.length > 0 || faildUpdateArchives.length > 0){ this.showDialog({ title: `Update Archive Error`, content: `Bookmarks Update Archive Faild : ${faildUpdateArchives.join(", ")} @@ -727,6 +727,7 @@ export default { this.dialog.visible = false; }, }) + } }).catch(err => { this.selection = []; this.editMode = false; From 658c80fee749a4226e4f8244e2d7f3642aaaa0ad Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 02:05:53 +0330 Subject: [PATCH 11/36] update thumbnail based on last status of bookmark fix #524 --- internal/core/processing.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index 1da0654f3..dd9759586 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -66,6 +66,8 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, } // If this is HTML, parse for readable content + strID := strconv.Itoa(book.ID) + imgPath := fp.Join(req.DataDir, "thumb", strID) var imageURLs []string if strings.Contains(contentType, "text/html") { isReadable := readability.Check(readabilityCheckInput) @@ -101,6 +103,8 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Get image URL if article.Image != "" { imageURLs = append(imageURLs, article.Image) + } else { + os.Remove(imgPath) } if article.Favicon != "" { @@ -115,9 +119,6 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, } // Save article image to local disk - strID := strconv.Itoa(book.ID) - imgPath := fp.Join(req.DataDir, "thumb", strID) - for _, imageURL := range imageURLs { err = downloadBookImage(imageURL, imgPath) if err == nil { From 87b6861a81dc50f78631f37611cd0b0716812890 Mon Sep 17 00:00:00 2001 From: Monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 13:11:40 +0330 Subject: [PATCH 12/36] better warning massage Co-authored-by: Felipe Martin <812088+fmartingr@users.noreply.github.com> --- internal/view/assets/js/page/home.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/view/assets/js/page/home.js b/internal/view/assets/js/page/home.js index 347e55f4f..e44cf69f4 100644 --- a/internal/view/assets/js/page/home.js +++ b/internal/view/assets/js/page/home.js @@ -720,8 +720,8 @@ export default { this.showDialog({ title: `Update Archive Error`, content: `Bookmarks Update Archive Faild : ${faildUpdateArchives.join(", ")} - Bookmarks Create Ebook Faild: ${faildCreateEbook.join(",")} - We recovered the last available version.`, + Bookmarks Create Ebook Failed: ${faildCreateEbook.join(",")} + Files that failed retrieval were not overwritten.`, mainText: "OK", mainClick: () => { this.dialog.visible = false; From c54da2637aa12681bca10ace3a8e13db7efee54f Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 13:27:42 +0330 Subject: [PATCH 13/36] tmpFile without .epub --- internal/core/ebook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index 7f4281daf..d767783dd 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -47,7 +47,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err } // create temporary epub file - tmpFile, err := os.CreateTemp("", "ebook*.epub") + tmpFile, err := os.CreateTemp("", "ebook") if err != nil { return book, errors.Wrap(err, "can't create temporary EPUB file") } From 0a752e182e47ff24b21075a23c38960ad2f38c34 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 13:34:13 +0330 Subject: [PATCH 14/36] MoveToDestination change to MoveFileToDestination --- internal/core/ebook.go | 2 +- internal/core/processing.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/core/ebook.go b/internal/core/ebook.go index d767783dd..6df619d47 100644 --- a/internal/core/ebook.go +++ b/internal/core/ebook.go @@ -227,7 +227,7 @@ img { } defer tmpFile.Close() - err = MoveToDestination(dstPath, tmpFile) + err = MoveFileToDestination(dstPath, tmpFile) if err != nil { return book, errors.Wrap(err, "failed move ebook to destination") } diff --git a/internal/core/processing.go b/internal/core/processing.go index dd9759586..aaaf1f726 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -167,7 +167,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Prepare destination file. dstPath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID)) - err = MoveToDestination(dstPath, tmpFile) + err = MoveFileToDestination(dstPath, tmpFile) if err != nil { return book, false, fmt.Errorf("failed move archive to destination `: %v", err) } @@ -250,7 +250,7 @@ func downloadBookImage(url, dstPath string) error { return fmt.Errorf("failed to save image %s: %v", url, err) } - err = MoveToDestination(dstPath, tmpFile) + err = MoveFileToDestination(dstPath, tmpFile) if err != nil { return err } @@ -258,7 +258,8 @@ func downloadBookImage(url, dstPath string) error { return nil } -func MoveToDestination(dstPath string, tmpFile *os.File) error { +// dstPath requires the filename +func MoveFileToDestination(dstPath string, tmpFile *os.File) error { // Prepare destination file. err := os.MkdirAll(fp.Dir(dstPath), model.DataDirPerm) if err != nil { From 89c04d69390d328fb22b1714dc580b8b027afe59 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 14:01:50 +0330 Subject: [PATCH 15/36] return .epub --- internal/core/processing.go | 2 +- internal/webserver/handler-api.go | 6 +++--- internal/webserver/handler-ui.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index aaaf1f726..ab866fe12 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -129,7 +129,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // If needed, create ebook as well if book.CreateEbook { - ebookPath := fp.Join(req.DataDir, "ebook", strID) + ebookPath := fp.Join(req.DataDir, "ebook", strID+".epub") if strings.Contains(contentType, "application/pdf") { return book, false, errors.Wrap(err, "can't create ebook from pdf") diff --git a/internal/webserver/handler-api.go b/internal/webserver/handler-api.go index cbbeb81b0..e4f6278a8 100644 --- a/internal/webserver/handler-api.go +++ b/internal/webserver/handler-api.go @@ -110,7 +110,7 @@ func (h *Handler) ApiGetBookmarks(w http.ResponseWriter, r *http.Request, ps htt strID := strconv.Itoa(bookmarks[i].ID) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) - ebookPath := fp.Join(h.DataDir, "ebook", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") if fileExists(imgPath) { bookmarks[i].ImageURL = path.Join(h.RootPath, "bookmark", strID, "thumb") @@ -285,7 +285,7 @@ func (h *Handler) ApiDeleteBookmark(w http.ResponseWriter, r *http.Request, ps h strID := strconv.Itoa(id) imgPath := fp.Join(h.DataDir, "thumb", strID) archivePath := fp.Join(h.DataDir, "archive", strID) - ebookPath := fp.Join(h.DataDir, "ebook", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") os.Remove(imgPath) os.Remove(archivePath) @@ -436,7 +436,7 @@ func (h *Handler) ApiDownloadEbook(w http.ResponseWriter, r *http.Request, ps ht // if file exist book return avilable file strID := strconv.Itoa(book.ID) - ebookPath := fp.Join(request.DataDir, "ebook", strID) + ebookPath := fp.Join(request.DataDir, "ebook", strID+".epub") _, err = os.Stat(ebookPath) if err == nil { // file already exists, return the existing file diff --git a/internal/webserver/handler-ui.go b/internal/webserver/handler-ui.go index 17800d169..712744f51 100644 --- a/internal/webserver/handler-ui.go +++ b/internal/webserver/handler-ui.go @@ -49,7 +49,7 @@ func (h *Handler) ServeBookmarkContent(w http.ResponseWriter, r *http.Request, p } // Check if it has ebook. - ebookPath := fp.Join(h.DataDir, "ebook", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") if fileExists(ebookPath) { bookmark.HasEbook = true } @@ -320,7 +320,7 @@ func (h *Handler) ServeBookmarkEbook(w http.ResponseWriter, r *http.Request, ps } // Check if it has ebook. - ebookPath := fp.Join(h.DataDir, "ebook", strID) + ebookPath := fp.Join(h.DataDir, "ebook", strID+".epub") if !fileExists(ebookPath) { http.Error(w, "ebook not found", http.StatusNotFound) return From 2b489ea7585d83afe4166d56dfbbebe5081dd114 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 14:37:34 +0330 Subject: [PATCH 16/36] log if downloadBookImage return error --- internal/core/processing.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index ab866fe12..de5aef868 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -8,6 +8,7 @@ import ( "image/draw" "image/jpeg" "io" + "log" "math" "net/url" "os" @@ -121,6 +122,14 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Save article image to local disk for _, imageURL := range imageURLs { err = downloadBookImage(imageURL, imgPath) + if err != nil { + if err.Error() == fmt.Sprintf("%s is not a supported image", imageURL) { + log.Printf("Not found image for URL: %s", imageURL) + } else { + log.Printf("File download not successful for image URL: %s", imageURL) + } + continue + } if err == nil { book.ImageURL = path.Join("/", "bookmark", strID, "thumb") break @@ -192,7 +201,7 @@ func downloadBookImage(url, dstPath string) error { !strings.Contains(cp, "image/pjpeg") && !strings.Contains(cp, "image/jpg") && !strings.Contains(cp, "image/png") { - + os.Remove(dstPath) return fmt.Errorf("%s is not a supported image", url) } From 358811946a1ea756cfbe21b56264b66b63c90b17 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 15:10:56 +0330 Subject: [PATCH 17/36] fix bug remove imgPath just if download last image be unsuccessful --- internal/core/processing.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/core/processing.go b/internal/core/processing.go index de5aef868..8350a4f9e 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -120,11 +120,14 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, } // Save article image to local disk - for _, imageURL := range imageURLs { + for i, imageURL := range imageURLs { err = downloadBookImage(imageURL, imgPath) if err != nil { if err.Error() == fmt.Sprintf("%s is not a supported image", imageURL) { log.Printf("Not found image for URL: %s", imageURL) + if i == len(imageURLs)-1 { + os.Remove(imgPath) + } } else { log.Printf("File download not successful for image URL: %s", imageURL) } @@ -201,7 +204,6 @@ func downloadBookImage(url, dstPath string) error { !strings.Contains(cp, "image/pjpeg") && !strings.Contains(cp, "image/jpg") && !strings.Contains(cp, "image/png") { - os.Remove(dstPath) return fmt.Errorf("%s is not a supported image", url) } From 81703cb3f68af824d6d5d69a02bfd538c2f40353 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Sun, 6 Aug 2023 20:34:29 +0330 Subject: [PATCH 18/36] update old unit test --- internal/core/ebook_test.go | 73 +++++++++---------------------------- 1 file changed, 18 insertions(+), 55 deletions(-) diff --git a/internal/core/ebook_test.go b/internal/core/ebook_test.go index 77c7bf15f..d4523b14c 100644 --- a/internal/core/ebook_test.go +++ b/internal/core/ebook_test.go @@ -1,7 +1,6 @@ package core_test import ( - "errors" "fmt" "os" fp "path/filepath" @@ -14,8 +13,9 @@ import ( func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testing.T) { tempDir := t.TempDir() - defer os.RemoveAll(tempDir) + parentDir := t.TempDir() + defer os.RemoveAll(parentDir) mockRequest := core.ProcessRequest{ Bookmark: model.Bookmark{ @@ -24,11 +24,11 @@ func TestGenerateEbook_ValidBookmarkID_ReturnsBookmarkWithHasEbookTrue(t *testin HTML: "
Example HTML", HasEbook: false, }, - DataDir: tempDir, + DataDir: parentDir, ContentType: "text/html", } - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) assert.True(t, bookmark.HasEbook) assert.NoError(t, err) @@ -46,7 +46,7 @@ func TestGenerateEbook_InvalidBookmarkID_ReturnsError(t *testing.T) { ContentType: "text/html", } - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, tempDir) assert.Equal(t, model.Bookmark{ ID: 0, @@ -58,13 +58,15 @@ func TestGenerateEbook_InvalidBookmarkID_ReturnsError(t *testing.T) { func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookTrue(t *testing.T) { tempDir := t.TempDir() defer os.RemoveAll(tempDir) + parentDir := t.TempDir() + defer os.RemoveAll(parentDir) mockRequest := core.ProcessRequest{ Bookmark: model.Bookmark{ ID: 1, HasEbook: false, }, - DataDir: tempDir, + DataDir: parentDir, ContentType: "text/html", } // Create the ebook directory @@ -81,7 +83,7 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookT } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) assert.True(t, bookmark.HasEbook) assert.NoError(t, err) @@ -90,13 +92,15 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_EbookExist_ReturnWithHasEbookT func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEbookTrue(t *testing.T) { tempDir := t.TempDir() defer os.RemoveAll(tempDir) + parentDir := t.TempDir() + defer os.RemoveAll(parentDir) mockRequest := core.ProcessRequest{ Bookmark: model.Bookmark{ ID: 1, HasEbook: false, }, - DataDir: tempDir, + DataDir: parentDir, ContentType: "text/html", } // Create the image directory @@ -113,7 +117,7 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEb } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) expectedimagePath := "/bookmark/1/thumb" if expectedimagePath != bookmark.ImageURL { t.Errorf("Expected imageURL %s, but got %s", bookmark.ImageURL, expectedimagePath) @@ -125,13 +129,15 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_ImagePathExist_ReturnWithHasEb func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *testing.T) { tempDir := t.TempDir() defer os.RemoveAll(tempDir) + parentDir := t.TempDir() + defer os.RemoveAll(parentDir) mockRequest := core.ProcessRequest{ Bookmark: model.Bookmark{ ID: 1, HasEbook: false, }, - DataDir: tempDir, + DataDir: parentDir, ContentType: "text/html", } // Create the archive directory @@ -148,7 +154,7 @@ func TestGenerateEbook_ValidBookmarkID_EbookExist_ReturnWithHasArchiveTrue(t *te } defer file.Close() - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, fp.Join(tempDir, "1")) assert.True(t, bookmark.HasArchive) assert.NoError(t, err) } @@ -166,56 +172,13 @@ func TestGenerateEbook_ValidBookmarkID_RetuenError_PDF(t *testing.T) { ContentType: "application/pdf", } - bookmark, err := core.GenerateEbook(mockRequest) + bookmark, err := core.GenerateEbook(mockRequest, tempDir) assert.False(t, bookmark.HasEbook) assert.Error(t, err) assert.Contains(t, err.Error(), "can't create ebook for pdf") } -func TestGenerateEbook_CreateEbookDirectoryNotWritable(t *testing.T) { - // Create a temporary directory to use as the parent directory - parentDir := t.TempDir() - - // Create a child directory with read-only permissions - ebookDir := fp.Join(parentDir, "ebook") - err := os.Mkdir(ebookDir, 0444) - if err != nil { - t.Fatalf("could not create ebook directory: %s", err) - } - - mockRequest := core.ProcessRequest{ - Bookmark: model.Bookmark{ - ID: 1, - HasEbook: false, - }, - DataDir: ebookDir, - ContentType: "text/html", - } - - // Call GenerateEbook to create the ebook directory - bookmark, err := core.GenerateEbook(mockRequest) - if err == nil { - t.Fatal("GenerateEbook succeeded even though MkdirAll should have failed") - } - if !errors.Is(err, os.ErrPermission) { - t.Fatalf("unexpected error: expected os.ErrPermission, got %v", err) - } - - // Check if the ebook directory still exists and has read-only permissions - info, err := os.Stat(ebookDir) - if err != nil { - t.Fatalf("could not retrieve ebook directory info: %s", err) - } - if !info.IsDir() { - t.Errorf("ebook directory is not a directory") - } - if info.Mode().Perm() != 0444 { - t.Errorf("ebook directory has incorrect permissions: expected 0444, got %o", info.Mode().Perm()) - } - assert.False(t, bookmark.HasEbook) -} - // Add more unit tests for other scenarios that missing specialy // can't create ebook directory and can't write situatuin // writing inside zip file From e56ef8fa0a981cf802db96a88a04c39eb4404687 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Mon, 7 Aug 2023 00:52:41 +0330 Subject: [PATCH 19/36] add processing.go unit test --- internal/core/processing.go | 4 +- internal/core/processing_test.go | 281 +++++++++++++++++++++++++++++++ 2 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 internal/core/processing_test.go diff --git a/internal/core/processing.go b/internal/core/processing.go index 8350a4f9e..b359a1ce8 100644 --- a/internal/core/processing.go +++ b/internal/core/processing.go @@ -121,7 +121,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, // Save article image to local disk for i, imageURL := range imageURLs { - err = downloadBookImage(imageURL, imgPath) + err = DownloadBookImage(imageURL, imgPath) if err != nil { if err.Error() == fmt.Sprintf("%s is not a supported image", imageURL) { log.Printf("Not found image for URL: %s", imageURL) @@ -190,7 +190,7 @@ func ProcessBookmark(req ProcessRequest) (book model.Bookmark, isFatalErr bool, return book, false, nil } -func downloadBookImage(url, dstPath string) error { +func DownloadBookImage(url, dstPath string) error { // Fetch data from URL resp, err := httpClient.Get(url) if err != nil { diff --git a/internal/core/processing_test.go b/internal/core/processing_test.go new file mode 100644 index 000000000..0c6001dd7 --- /dev/null +++ b/internal/core/processing_test.go @@ -0,0 +1,281 @@ +package core_test + +import ( + "bytes" + "os" + fp "path/filepath" + "testing" + + "github.com/go-shiori/shiori/internal/core" + "github.com/go-shiori/shiori/internal/model" + "github.com/stretchr/testify/assert" +) + +func TestMoveFileToDestination_CreateDir_Fails(t *testing.T) { + tmpFile, err := os.CreateTemp("", "image") + + assert.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + err = core.MoveFileToDestination("/destination/test", tmpFile) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to create destination dir") +} + +func TestMoveFileToDestination_CreateFile_Fails(t *testing.T) { + tmpFile, err := os.CreateTemp("", "image") + assert.NoError(t, err) + defer os.Remove(tmpFile.Name()) + + // Create a destination directory + dstDir := t.TempDir() + assert.NoError(t, err) + defer os.Remove(dstDir) + + // Set destination path to an invalid file name to force os.Create to fail + dstPath := fp.Join(dstDir, "\000invalid\000") + + err = core.MoveFileToDestination(dstPath, tmpFile) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to create destination file") +} + +// TestDownloadBookImage_Success tests the DownloadBookImage function with a valid image URL. +func TestDownloadBookImage_notSuccess(t *testing.T) { + // Arrange + imageURL := "https://github.com/go-shiori/shiori/blob/master/internal/view/assets/res/apple-touch-icon-152x152.png" + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + dstPath := fp.Join(tempDir, "1") + defer os.Remove(dstPath) + + // Act + err := core.DownloadBookImage(imageURL, dstPath) + + // Assert + //assert.NoError(t, err) + assert.EqualError(t, err, "https://github.com/go-shiori/shiori/blob/master/internal/view/assets/res/apple-touch-icon-152x152.png is not a supported image") + assert.NoFileExists(t, dstPath) +} + +func TestDownloadBookImage_Success(t *testing.T) { + // Arrange + imageURL := "https://raw.githubusercontent.com/go-shiori/shiori/master/docs/readme/cover.png" + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + dstPath := fp.Join(tempDir, "1") + defer os.Remove(dstPath) + + // Act + err := core.DownloadBookImage(imageURL, dstPath) + + // Assert + assert.NoError(t, err) + assert.FileExists(t, dstPath) +} + +func TestDownloadBookImage_SuccessSmallSize(t *testing.T) { + // Arrange + imageURL := "https://www.google.com/images/branding/googlelogo/2x/googlelogo_light_color_272x92dp.png" + tempDir := t.TempDir() + defer os.RemoveAll(tempDir) + dstPath := fp.Join(tempDir, "1") + defer os.Remove(dstPath) + + // Act + err := core.DownloadBookImage(imageURL, dstPath) + + // Assert + assert.NoError(t, err) + assert.FileExists(t, dstPath) +} + +func TestProcessBookmark(t *testing.T) { + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "Example", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString("This is an example article
") + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + expected, _, _ := core.ProcessBookmark(request) + + if expected.ID != bookmark.ID { + t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) + } + if expected.URL != bookmark.URL { + t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL) + } + if expected.Title != bookmark.Title { + t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title) + } + if expected.Excerpt != bookmark.Excerpt { + t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) + } +} + +func TestProcessBookmarkMultipleImage(t *testing.T) { + html := `html + + + + + + +This is an example article
+ +` + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "Example", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString(html) + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + expected, _, _ := core.ProcessBookmark(request) + + if expected.ID != bookmark.ID { + t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) + } + if expected.URL != bookmark.URL { + t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL) + } + if expected.Title != bookmark.Title { + t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title) + } + if expected.Excerpt != bookmark.Excerpt { + t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) + } +} + +func TestProcessBookmarkMultipleImagefaveiconAndThumb(t *testing.T) { + html := `html + + + + + + +This is an example article
+ +` + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "Example", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString(html) + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + expected, _, _ := core.ProcessBookmark(request) + + if expected.ID != bookmark.ID { + t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) + } + if expected.URL != bookmark.URL { + t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL) + } + if expected.Title != bookmark.Title { + t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title) + } + if expected.Excerpt != bookmark.Excerpt { + t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) + } +} + +func TestProcessBookmarkEmptyTitle(t *testing.T) { + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString("This is an example article
") + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + expected, _, _ := core.ProcessBookmark(request) + + if expected.ID != bookmark.ID { + t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) + } + if expected.URL != bookmark.URL { + t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL) + } + if expected.Title != bookmark.URL { + t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title) + } + if expected.Excerpt != bookmark.Excerpt { + t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) + } +} + +func TestProcessBookmarkKeepExcerptEmpty(t *testing.T) { + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString("This is an example article
") + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: false, + } + expected, _, _ := core.ProcessBookmark(request) + + if expected.ID != bookmark.ID { + t.Errorf("Unexpected ID: got %v, want %v", expected.ID, bookmark.ID) + } + if expected.URL != bookmark.URL { + t.Errorf("Unexpected URL: got %v, want %v", expected.URL, bookmark.URL) + } + if expected.Title != bookmark.URL { + t.Errorf("Unexpected Title: got %v, want %v", expected.Title, bookmark.Title) + } + if expected.Excerpt != bookmark.Excerpt { + t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) + } +} From 947eefa80dd544daecdd656001fbcd1447f99803 Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Mon, 7 Aug 2023 01:18:22 +0330 Subject: [PATCH 20/36] small massage for report failded item to the user --- internal/view/assets/js/page/home.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/view/assets/js/page/home.js b/internal/view/assets/js/page/home.js index e44cf69f4..4b0d5acc0 100644 --- a/internal/view/assets/js/page/home.js +++ b/internal/view/assets/js/page/home.js @@ -703,24 +703,24 @@ export default { this.dialog.loading = false; this.dialog.visible = false; - let faildUpdateArchives = []; - let faildCreateEbook = []; + let faildedUpdateArchives = []; + let faildedCreateEbook = []; json.forEach(book => { var item = items.find(el => el.id === book.id); this.bookmarks.splice(item.index, 1, book); if (data.createArchive && !book.hasArchive){ - faildUpdateArchives.push(book.id); + faildedUpdateArchives.push(book.id); } if (data.createEbook && !book.hasEbook){ - faildCreateEbook.push(book.id); + faildedCreateEbook.push(book.id); } }); if(faildCreateEbook.length > 0 || faildUpdateArchives.length > 0){ this.showDialog({ - title: `Update Archive Error`, - content: `Bookmarks Update Archive Faild : ${faildUpdateArchives.join(", ")} - Bookmarks Create Ebook Failed: ${faildCreateEbook.join(",")} + title: `Bookmarks Id that Update Action Faild`, + content: `Archive:[ ${faildedUpdateArchives.join(", ")} ] + Ebook: [ ${faildedCreateEbook.join(",")} ] Files that failed retrieval were not overwritten.`, mainText: "OK", mainClick: () => { From 757599fcfc729bffc28e007adf71400248885d0b Mon Sep 17 00:00:00 2001 From: monirzadeh <25131576+Monirzadeh@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:28:54 +0330 Subject: [PATCH 21/36] add some more unit test and samplefile --- internal/core/processing_test.go | 48 +++++++++++++++++++++++++++++-- testdata/big_image.png | Bin 0 -> 16174 bytes testdata/favicon.png | Bin 0 -> 534 bytes testdata/favicon.svg | 6 ++++ testdata/medium_image.png | Bin 0 -> 3340 bytes 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 testdata/big_image.png create mode 100644 testdata/favicon.png create mode 100644 testdata/favicon.svg create mode 100644 testdata/medium_image.png diff --git a/internal/core/processing_test.go b/internal/core/processing_test.go index 0c6001dd7..e029f2b02 100644 --- a/internal/core/processing_test.go +++ b/internal/core/processing_test.go @@ -53,7 +53,6 @@ func TestDownloadBookImage_notSuccess(t *testing.T) { err := core.DownloadBookImage(imageURL, dstPath) // Assert - //assert.NoError(t, err) assert.EqualError(t, err, "https://github.com/go-shiori/shiori/blob/master/internal/view/assets/res/apple-touch-icon-152x152.png is not a supported image") assert.NoFileExists(t, dstPath) } @@ -173,7 +172,7 @@ func TestProcessBookmarkMultipleImagefaveiconAndThumb(t *testing.T) { - +This is an example article
@@ -279,3 +278,48 @@ func TestProcessBookmarkKeepExcerptEmpty(t *testing.T) { t.Errorf("Unexpected Excerpt: got %v, want %v", expected.Excerpt, bookmark.Excerpt) } } + +func TestProcessBookmarkIDZero(t *testing.T) { + bookmark := model.Bookmark{ + ID: 0, + URL: "https://example.com", + Title: "Example", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString("This is an example article
") + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "text/html", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + _, isFatal, err := core.ProcessBookmark(request) + assert.Error(t, err) + assert.Contains(t, err.Error(), "bookmark ID is not valid") + assert.True(t, isFatal) +} +func TestProcessBookmarkContentTypeNotTextHtml(t *testing.T) { + bookmark := model.Bookmark{ + ID: 1, + URL: "https://example.com", + Title: "Example", + Excerpt: "This is an example article", + CreateEbook: true, + CreateArchive: true, + } + content := bytes.NewBufferString("This is an example article
") + request := core.ProcessRequest{ + Bookmark: bookmark, + Content: content, + ContentType: "application/pdf", + DataDir: "/tmp", + KeepTitle: true, + KeepExcerpt: true, + } + _, _, err := core.ProcessBookmark(request) + assert.NoError(t, err) +} diff --git a/testdata/big_image.png b/testdata/big_image.png new file mode 100644 index 0000000000000000000000000000000000000000..e1a2372628f0fecf017f025e3bd5ccf5198a0ea3 GIT binary patch literal 16174 zcmeHtX;@QNyLQ?hO9zp+wH1*<-nTNer3wgw3`t5=h9Xe4N*SYqfDI5CL*cm7+ju zQbDGmQlXWppb#J=ky1&w3=vVLkOW9%3Q0&9GAC!p@4de3oFC`k`E@SVpS`k^wf0)i zx}W=go@b}<>+m4UclNvkfj}%nf{&blKrF67AaAt(<3GSVnv3vh@bb@u;1gjGNWK#U za_I*ML<`=!vV_#v^D+V-_KEst)L(
z@6eX~mQprKsqLuB$2Vaq*G;~Q{IK}Lhpz~FO`6SUfa6Nb*9S#9+qO&@tJ00o8#q}+
zX>97s!W$6CvC}7C)nd*}WzWn5W!yz!K0EK^0VL&9or6hpKT$#6q22L&;*G0ZUUcff
zNK@a;=h^*>{nm(9M$Q81>{HB}5QrQ6hjS3fnQ!7J{#ma1N&FZdI9HZW6eborNQS9;
zd|!ZY->E@-Sf9H8iok|np^=89*W}~tx_D5;0G{9EH!8cr@&LD|nDx8%8st)QTx-~Y
zm(IZTmOUm^+#e%i?YA*?S;%yR|9X1>x+vWeI_B1zpqst3w>vhkGC*hpfjm;L2D(Ea
zPj5 This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article This is an example article4ipV8SayJuA_>p7adBI~sAvL}GQ3N_tp
zZ@B_TXtOxHeWWyL+UGY8g&0@M6I&&e565lRtB)~uP5%mFqE=5hXfL>US~>3+2rZ
PNqX=qgJABk