Merge pull request #17 from jadkins89/addImageUrl

Adding images to scrapes
jadkins89 · Apr 11, 2020 · b9c64ac · b9c64ac
2 parents 4701ac3 + 06805fe
commit b9c64ac
Show file tree

Hide file tree

Showing 61 changed files with 193 additions and 86 deletions.
diff --git a/README.md b/README.md
@@ -76,6 +76,7 @@ Depending on the recipe, certain fields may be left blank. All fields are repres
     ingredients: [],
     instructions: [],
     servings: "",
+    image: "",
     time: {
       prep: "",
       cook: "",

diff --git a/helpers/puppeteerFetch.js b/helpers/puppeteerFetch.js
@@ -61,13 +61,15 @@ const puppeteerFetch = async url => {
     let html = await page.content();
     try {
       await browser.close();
-    } catch (error) {} // avoid websocket error if browser already closed
-    return html;
+    } finally {
+      return html;
+    } // avoid websocket error if browser already closed
   } else {
     try {
       await browser.close();
-    } catch (error) {}
-    return Promise.reject(response._status);
+    } finally {
+      return Promise.reject(response._status);
+    }
   }
 };
 

diff --git a/helpers/recipe-schema.js b/helpers/recipe-schema.js
@@ -11,6 +11,7 @@ function Recipe() {
     total: ""
   };
   this.servings = "";
+  this.image = "";
 }
 
 module.exports = Recipe;
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "recipe-scraper",
-  "version": "1.14.1",
+  "version": "1.15.0",
   "description": "A JS package for scraping recipes from the web.",
   "author": "Justin Adkins <[email protected]>",
   "license": "MIT",

diff --git a/scrapers/101cookbooks.js b/scrapers/101cookbooks.js
@@ -14,6 +14,7 @@ const oneHundredAndOne = url => {
           const $ = cheerio.load(html);
           const body = $(".wprm-recipe-container");
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = body.children("h2").text();
 
           $(".wprm-recipe-ingredient").each((i, el) => {

diff --git a/scrapers/allrecipes.js b/scrapers/allrecipes.js
@@ -30,6 +30,7 @@ const allRecipes = url => {
 };
 
 const newAllRecipes = ($, Recipe) => {
+  Recipe.image = $("meta[property='og:image']").attr("content");
   Recipe.name = Recipe.name.replace(/\s\s+/g, "");
 
   $(".recipe-meta-item").each((i, el) => {
@@ -76,6 +77,8 @@ const newAllRecipes = ($, Recipe) => {
 };
 
 const oldAllRecipes = ($, Recipe) => {
+  Recipe.image = $("meta[property='og:image']").attr("content");
+
   $("#polaris-app label").each((i, el) => {
     const item = $(el)
       .text()

diff --git a/scrapers/ambitiouskitchen.js b/scrapers/ambitiouskitchen.js
@@ -13,6 +13,7 @@ const ambitiousKitchen = url => {
         const Recipe = new RecipeSchema();
         const $ = cheerio.load(html);
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(".wprm-recipe-name").text();
 
         $(".wprm-recipe-ingredient").each((i, el) => {

diff --git a/scrapers/bbc.js b/scrapers/bbc.js
@@ -13,6 +13,7 @@ const bbc = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".content-title__text").text();
 
           $(".recipe-ingredients__list-item").each((i, el) => {

diff --git a/scrapers/bbcgoodfood.js b/scrapers/bbcgoodfood.js
@@ -13,6 +13,7 @@ const bbcGoodFood = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe-header__title").text();
 
           $(".ingredients-list__item").each((i, el) => {

diff --git a/scrapers/bonappetit.js b/scrapers/bonappetit.js
@@ -13,6 +13,7 @@ const bonAppetit = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $("a.top-anchor").text();
 
           $(".ingredients")

diff --git a/scrapers/budgetbytes.js b/scrapers/budgetbytes.js
@@ -13,6 +13,7 @@ const budgetBytes = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".wprm-recipe-name").text();
 
           $(".wprm-recipe-ingredient-notes").remove();

diff --git a/scrapers/closetcooking.js b/scrapers/closetcooking.js
@@ -13,6 +13,7 @@ const closetCooking = url => {
         const Recipe = new RecipeSchema();
         const $ = cheerio.load(html);
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(".recipe_title").text();
 
         $(".ingredients")

diff --git a/scrapers/cookieandkate.js b/scrapers/cookieandkate.js
@@ -13,6 +13,7 @@ const cookieAndKate = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".tasty-recipes")
             .children("h2")
             .text();

diff --git a/scrapers/copykat.js b/scrapers/copykat.js
@@ -13,6 +13,7 @@ const copykat = url => {
         var Recipe = new RecipeSchema();
         const $ = cheerio.load(html);
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(
           $(".wprm-recipe-container").find(".wprm-recipe-name")
         ).text();

diff --git a/scrapers/damndelicious.js b/scrapers/damndelicious.js
@@ -15,6 +15,7 @@ const damnDelicious = url => {
 
         let titleDiv = $(".recipe-title");
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(titleDiv)
           .children("h2")
           .text();

diff --git a/scrapers/eatingwell.js b/scrapers/eatingwell.js
@@ -13,6 +13,7 @@ const eatingWell = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".main-header")
             .find(".headline")
             .text()

diff --git a/scrapers/epicurious.js b/scrapers/epicurious.js
@@ -13,6 +13,7 @@ const epicurious = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $("h1[itemprop=name]")
             .text()
             .trim();

diff --git a/scrapers/finecooking.js b/scrapers/finecooking.js
@@ -13,6 +13,7 @@ const fineCooking = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe__title").text();
 
           $(".recipe__nutrition").remove();

diff --git a/scrapers/food.js b/scrapers/food.js
@@ -13,6 +13,7 @@ const food = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[name='og:image']").attr("content");
           Recipe.name = $(".recipe-title").text();
 
           $(".recipe-ingredients__item").each((i, el) => {

diff --git a/scrapers/foodandwine.js b/scrapers/foodandwine.js
@@ -13,6 +13,7 @@ const foodAndWine = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $("h1.headline").text();
 
           $(".ingredients")

diff --git a/scrapers/foodnetwork.js b/scrapers/foodnetwork.js
@@ -13,9 +13,11 @@ const foodNetwork = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".o-AssetTitle__a-HeadlineText")
             .first()
             .text();
+
           $(".o-Ingredients__a-Ingredient, .o-Ingredients__a-SubHeadline").each(
             (i, el) => {
               const item = $(el)

diff --git a/scrapers/gimmesomeoven.js b/scrapers/gimmesomeoven.js
@@ -13,6 +13,7 @@ const gimmeSomeOven = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".tasty-recipes-header-content")
             .children("h2")
             .first()

diff --git a/scrapers/kitchenstories.js b/scrapers/kitchenstories.js
@@ -20,6 +20,7 @@ const kitchenStories = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe-title").text();
 
           $(".ingredients")

diff --git a/scrapers/myrecipes.js b/scrapers/myrecipes.js
@@ -13,6 +13,7 @@ const myRecipes = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $("h1.headline")
             .text()
             .trim();

diff --git a/scrapers/seriouseats.js b/scrapers/seriouseats.js
@@ -39,6 +39,7 @@ const seriousEats = url => {
 };
 
 const regularRecipe = ($, Recipe) => {
+  Recipe.image = $("meta[property='og:image']").attr("content");
   Recipe.name = $(".recipe-title")
     .text()
     .replace(/\s\s+/g, "");

diff --git a/scrapers/simplyrecipes.js b/scrapers/simplyrecipes.js
@@ -15,6 +15,7 @@ const simplyRecipes = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe-callout")
             .children("h2")
             .text();

diff --git a/scrapers/smittenkitchen.js b/scrapers/smittenkitchen.js
@@ -42,6 +42,7 @@ const oldSmitten = ($, Recipe) => {
   let servingWords = ["Yield", "Serve", "Servings"];
   let servingsRegex = new RegExp(servingWords.join("|"), "i");
 
+  Recipe.image = $("meta[property='og:image']").attr("content");
   body.each((i, el) => {
     if (i === 0) {
       Recipe.name = $(el)
@@ -84,6 +85,7 @@ const oldSmitten = ($, Recipe) => {
 };
 
 const newSmitten = ($, Recipe) => {
+  Recipe.image = $("meta[property='og:image']").attr("content");
   Recipe.name = $(".jetpack-recipe-title").text();
 
   $(".jetpack-recipe-ingredients")

diff --git a/scrapers/thepioneerwoman.js b/scrapers/thepioneerwoman.js
@@ -15,6 +15,7 @@ const thePioneerWoman = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe-title")
             .first()
             .text();

diff --git a/scrapers/therealfoodrds.js b/scrapers/therealfoodrds.js
@@ -15,6 +15,7 @@ const theRealFoodRds = url => {
         const Recipe = new RecipeSchema();
         const $ = cheerio.load(html);
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(".tasty-recipes-entry-header")
           .children("h2")
           .first()

diff --git a/scrapers/thespruceeats.js b/scrapers/thespruceeats.js
@@ -13,6 +13,7 @@ const theSpruceEats = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".heading__title").text();
 
           $(".simple-list__item").each((i, el) => {

diff --git a/scrapers/whatsgabycooking.js b/scrapers/whatsgabycooking.js
@@ -13,6 +13,7 @@ const whatsGabyCooking = url => {
         if (!error && response.statusCode === 200) {
           const $ = cheerio.load(html);
 
+          Recipe.image = $("meta[property='og:image']").attr("content");
           Recipe.name = $(".recipe-header").text();
 
           $(".wprm-recipe-ingredient").each((i, el) => {

diff --git a/scrapers/woolworths.js b/scrapers/woolworths.js
@@ -1,4 +1,5 @@
 const request = require("request");
+const cheerio = require("cheerio");
 
 const RecipeSchema = require("../helpers/recipe-schema");
 
@@ -25,17 +26,20 @@ const woolworths = url => {
           url: recipeJsonUrl,
           json: true
         },
-        (error, response, body) => {
-          if (!error && response.statusCode === 200 && body) {
-            Recipe.name = body.Title.trim();
-            Recipe.ingredients = body.Ingredients.map(i =>
+        (error, response, html) => {
+          if (!error && response.statusCode === 200 && html) {
+            const $ = cheerio.load(html);
+
+            Recipe.image = $("meta[property='og:image']").attr("content");
+            Recipe.name = html.Title.trim();
+            Recipe.ingredients = html.Ingredients.map(i =>
               i.Description.trim()
             );
-            Recipe.time.prep = body.PreparationDuration.toString();
-            Recipe.time.cook = body.CookingDuration.toString();
-            Recipe.servings = body.Servings.toString();
+            Recipe.time.prep = html.PreparationDuration.toString();
+            Recipe.time.cook = html.CookingDuration.toString();
+            Recipe.servings = html.Servings.toString();
             Recipe.instructions = [];
-            body.Instructions.split("\r\n").map(step => {
+            html.Instructions.split("\r\n").map(step => {
               if (instructionsIndexRe.test(step)) {
                 Recipe.instructions.push(
                   instructionsIndexRe.exec(step)[1].trim()

diff --git a/scrapers/yummly.js b/scrapers/yummly.js
@@ -94,6 +94,7 @@ const yummy = url => {
         const Recipe = new RecipeSchema();
         const $ = cheerio.load(html);
 
+        Recipe.image = $("meta[property='og:image']").attr("content");
         Recipe.name = $(".recipe-title").text();
 
         $(".IngredientLine").each((i, el) => {

diff --git a/test/constants/101cookbooksConstants.js b/test/constants/101cookbooksConstants.js
@@ -32,6 +32,7 @@ module.exports = {
       ready: "",
       total: "15 mins"
     },
-    servings: "8"
+    servings: "8",
+    image: "https://images.101cookbooks.com/coleslaw-recipe-h.jpg?w=680"
   }
 };
diff --git a/test/constants/allRecipesConstants.js b/test/constants/allRecipesConstants.js
@@ -29,7 +29,8 @@ module.exports = {
       ready: "25 m",
       total: ""
     },
-    servings: "6"
+    servings: "6",
+    image: "https://images.media-allrecipes.com/userphotos/560x315/2253389.jpg"
   },
   expectedRecipeNew: {
     name: "Crispy and Tender Baked Chicken Thighs",
@@ -57,6 +58,8 @@ module.exports = {
       ready: "",
       total: "1 hr 10 mins"
     },
-    servings: "8"
+    servings: "8",
+    image:
+      "https://imagesvc.meredithcorp.io/v3/mm/image?url=https%3A%2F%2Fimages.media-allrecipes.com%2Fuserphotos%2F1061355.jpg"
   }
 };
diff --git a/test/constants/ambitiouskitchenConstants.js b/test/constants/ambitiouskitchenConstants.js
@@ -51,6 +51,8 @@ module.exports = {
       ready: "",
       total: "45 minutes"
     },
-    servings: "6"
+    servings: "6",
+    image:
+      "https://www.ambitiouskitchen.com/wp-content/uploads/2018/07/Street-Corn-Pasta-Salad-4.jpg"
   }
 };
diff --git a/test/constants/bbcConstants.js b/test/constants/bbcConstants.js
@@ -27,6 +27,8 @@ module.exports = {
       ready: "",
       total: ""
     },
-    servings: "Serves 2"
+    servings: "Serves 2",
+    image:
+      "https://ichef.bbci.co.uk/food/ic/food_16x9_448/recipes/sausage_and_gnocchi_bake_80924_16x9.jpg"
   }
 };
diff --git a/test/constants/bbcgoodfoodConstants.js b/test/constants/bbcgoodfoodConstants.js
@@ -28,6 +28,8 @@ module.exports = {
       ready: "",
       total: ""
     },
-    servings: "Makes 12"
+    servings: "Makes 12",
+    image:
+      "https://www.bbcgoodfood.com/sites/default/files/recipe_images/recipe-image-legacy-id--856543_10.jpg"
   }
 };
diff --git a/test/constants/bonappetitConstants.js b/test/constants/bonappetitConstants.js
@@ -33,6 +33,8 @@ module.exports = {
       ready: "",
       total: ""
     },
-    servings: "4 servings"
+    servings: "4 servings",
+    image:
+      "https://assets.bonappetit.com/photos/5d4b5b3cecc81500091c6835/16:9/w_1200,c_limit/0919-Soba-Noodles.jpg"
   }
 };