From 9c7b5c722d9e70c9eeb4e7c29c437258e941db18 Mon Sep 17 00:00:00 2001 From: abdulrahman1s <61483023+abdulrahman1s@users.noreply.github.com> Date: Mon, 19 Sep 2022 22:25:17 +0200 Subject: [PATCH] feat: v0.1.0 --- .gitignore | 3 +- README.md | 20 +-- cookies.example.json | 8 -- package-lock.json | 3 +- package.json | 32 ++++- src/constants.js | 6 +- src/index.js | 309 +++++++++++++++++++++---------------------- src/polyfill.js | 19 +++ src/util/common.js | 16 ++- src/util/index.js | 3 - src/util/m3u8.js | 6 +- 11 files changed, 225 insertions(+), 200 deletions(-) delete mode 100644 cookies.example.json create mode 100644 src/polyfill.js delete mode 100644 src/util/index.js diff --git a/.gitignore b/.gitignore index 56bf2f5..b512c09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -node_modules -cookies.json +node_modules \ No newline at end of file diff --git a/README.md b/README.md index 0b8be11..429bf64 100644 --- a/README.md +++ b/README.md @@ -12,25 +12,17 @@ ### Requirements - Nodejs v16 or newer installed -- Git Installed -- FrontendMasters account +- Valid FrontendMasters account cookies ([Guide](https://developer.chrome.com/docs/devtools/storage/cookies/)) ### Usage -1. Clone the repo -```sh -$ git clone https://github.com/abdulrahman1s/fem-dl.git -$ cd fem-dl +```s +$ npx fem-dl ``` -2. Rename [`cookies.example.json`](./cookies.example.json) to `cookies.json` - -3. Copy the saved cookies at **frontendmasters.com** to `cookies.json` ([Guide](https://developer.chrome.com/docs/devtools/storage/cookies/)) - -4. Run! -```sh -$ npm install && npm start +or via pnpm +```s +$ pnpm dlx fem-dl ``` - ### Disclaimer I am not responsible for any use of this program, please read [FrontendMasters terms of service](https://static.frontendmasters.com/assets/legal/MasterServicesAgreement.pdf) before using this. diff --git a/cookies.example.json b/cookies.example.json deleted file mode 100644 index d92e133..0000000 --- a/cookies.example.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "name": "wordpress_logged_in_*other-random-id*", - "value": "super-long-secret-value", - "domain": ".frontendmasters.com", - "httpOnly": true - } -] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8571b8c..2fdb4c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,12 @@ { "name": "fem-dl", + "version": "0.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "fem-dl", - "license": "MIT", + "version": "0.0.1", "dependencies": { "ffmpeg-static": "^5.1.0", "kleur": "^4.1.5", diff --git a/package.json b/package.json index 17b0b4e..9b632a1 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,13 @@ { "name": "fem-dl", - "version": "0.0.1", + "version": "0.1.0", "description": "Frontend Masters Downloader", - "private": true, + "main": "src/index.js", "scripts": { - "start": "node --no-warnings --experimental-specifier-resolution=node src/index.js" + "start": "node src/index.js" + }, + "bin": { + "fem-dl": "src/index.js" }, "type": "module", "dependencies": { @@ -17,5 +20,24 @@ }, "engines": { "node": ">=16" - } -} + }, + "homepage": "https://github.com/abdulrahman1s/fem-dl#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/abdulrahman1s/fem-dl.git" + }, + "bugs": { + "url": "https://github.com/abdulrahman1s/fem-dl/issues" + }, + "license": "UNLICENSE", + "files": [ + "src/*" + ], + "keywords": [ + "frontendmasters", + "fem", + "downloader", + "cli", + "courses" + ] +} \ No newline at end of file diff --git a/src/constants.js b/src/constants.js index a79036c..81f0310 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,7 +1,9 @@ const FEM_BASE = 'frontendmasters.com' +export const FEM_ENDPOINT = `https://${FEM_BASE}` export const FEM_API_ENDPOINT = `https://api.${FEM_BASE}/v1` export const FEM_CAPTIONS_ENDPOINT = `https://captions.${FEM_BASE}` + export const PLAYLIST_EXT = 'm3u8' export const CAPTION_EXT = 'vtt' export const QUALITY_FORMAT = { @@ -12,7 +14,9 @@ export const QUALITY_FORMAT = { 360: 'index_360_Q8_2mbps' } +export const FEM_COURSE_REG = /(:?https?:\/\/)?frontendmasters\.com\/courses\/([^/]+)/ + export const SUPPORTED_FORMATS = [ 'mp4', - // 'mkv' + 'mkv' ] \ No newline at end of file diff --git a/src/index.js b/src/index.js index a582a9a..7654738 100644 --- a/src/index.js +++ b/src/index.js @@ -1,227 +1,218 @@ console.clear() -import { FEM_API_ENDPOINT, FEM_CAPTIONS_ENDPOINT, CAPTION_EXT, PLAYLIST_EXT, QUALITY_FORMAT } from './constants.js' -import { ffmpeg, sleep, isPathExists, ensureDir, get, M3u8 } from './util' -import { join } from 'node:path' +import './polyfill.js' +import { FEM_ENDPOINT, FEM_API_ENDPOINT, FEM_CAPTIONS_ENDPOINT, CAPTION_EXT, PLAYLIST_EXT, QUALITY_FORMAT, FEM_COURSE_REG, SUPPORTED_FORMATS } from './constants.js' +import { sleep, isPathExists, ensureDir, get, safeJoin } from './util/common.js' +import M3u8 from './util/m3u8.js' +import ffmpeg from './util/ffmpeg.js' import puppeteer from 'puppeteer' import fs from 'node:fs/promises' import prompts from 'prompts' import ora from 'ora' import colors from 'kleur' -import cookies from '../cookies.json' +import os from 'node:os' + +const exitOnCancel = (state) => { + if (state.aborted) process.nextTick(() => process.exit(0)) +} const { - URL, + COURSE_SLUG, QUALITY, DOWNLOAD_DIR, EXTENSION, - INCLUDE_CAPTION + INCLUDE_CAPTION, + COOKIES } = await prompts([{ type: 'text', - name: 'URL', - message: 'The url of the course you want to download (any lesson/section link)', - validate: v => v.startsWith('https://') + name: 'COURSE_SLUG', + message: 'The url of the course you want to download', + initial: 'https://frontendmasters.com/courses/...', + validate: v => !v.endsWith('...') && FEM_COURSE_REG.test(v), + format: v => v.match(FEM_COURSE_REG)[2], + onState: exitOnCancel +}, { + type: 'password', + name: 'COOKIES', + message: 'Paste the value of "wordpress_logged_in_xxx" cookie (visit: frontendmasters.com)', + format: value => ({ + name: 'wordpress_logged_in_323a64690667409e18476e5932ed231e', + value, + domain: '.frontendmasters.com', + httpOnly: true + }), + onState: exitOnCancel }, { type: 'select', name: 'QUALITY', - message: 'Which stream quality do you prefer? ', - choices: [ - { title: '2160p', value: 2160 }, - { title: '1440p', value: 1440 }, - { title: '1080p', value: 1080 }, - { title: '720p', value: 720 }, - { title: '360p', value: 360 } - ], - format: v => QUALITY_FORMAT[v] + message: 'Which stream quality do you prefer?', + choices: [2160, 1440, 1080, 720, 360].map((value) => ({ title: value + 'p', value })), + format: v => QUALITY_FORMAT[v], + onState: exitOnCancel }, { type: 'select', message: 'Which video format you prefer?', name: 'EXTENSION', initial: 0, - choices: [{ - title: 'mp4', - value: 'mp4' - }, { - title: 'mkv', - value: 'mkv' - }] + choices: SUPPORTED_FORMATS.map((value) => ({ title: value, value })), + onState: exitOnCancel }, { type: 'confirm', initial: true, name: 'INCLUDE_CAPTION', - message: 'Insert caption/subtitle to the episodes?' + message: 'Insert caption/subtitle to the episodes?', + onState: exitOnCancel }, { type: 'text', message: 'Download directory path', name: 'DOWNLOAD_DIR', - initial: process.cwd(), - validate: v => isPathExists(v) + initial: safeJoin(os.homedir(), 'Downloads'), + validate: v => isPathExists(v), + onState: exitOnCancel }]) console.clear() -const spinner = ora().start('Launching chrome engine...') -const browser = await puppeteer.launch({ - headless: true, - args: ["--fast-start", "--disable-extensions", "--no-sandbox"], -}) - -const page = await browser.newPage() +const + spinner = ora('Launching chromium engine...').start(), + browser = await puppeteer.launch({ + headless: true, + args: ['--fast-start', '--disable-extensions', '--no-sandbox'] + }), + page = await browser.newPage() -await page.setCookie(...cookies) +spinner.text = `Going to "${colors.underline().italic(FEM_ENDPOINT)}"` -spinner.text = `Going to "${colors.underline().italic(URL)}"` +await page.setCookie(COOKIES) +await page.goto(FEM_ENDPOINT + '/manifest.json') -await page.goto(URL) +spinner.text = 'Fetching course payload' -async function domFetch(url, type = 'text') { - let code +const course = await page.fetch(`${FEM_API_ENDPOINT}/kabuki/courses/${COURSE_SLUG}`, 'json') - if (type === 'text' || type === 'json') code = `fetch("${url}", { credentials: "include" }).then(r => r.${type}())` - // https://github.com/puppeteer/puppeteer/issues/3722 - else if (type === 'binary') code = `fetch("${url}", { credentials: "include" }).then(r => new Promise(async resolve => { - const reader = new FileReader(); - reader.readAsBinaryString(await r.blob()); - reader.onload = () => resolve(reader.result); - reader.onerror = () => reject('Error occurred while reading binary string'); - }))` - else throw new Error('Unknown type: ' + type) - - const result = await page.evaluate(code) - - return type === 'binary' ? Buffer.from(result, 'binary') : result +if (course.code === 404) { + spinner.fail(`Couldn't find this course "${COURSE_SLUG}"`) + process.exit() } - -spinner.text = 'Waiting for course payload' - -const response = await page.waitForResponse(res => { - if (spinner.text.endsWith('...')) spinner.text = spinner.text.slice(0, -3) - spinner.text += '.' - return res.request().method() === 'GET' && res.url().startsWith(`${FEM_API_ENDPOINT}/kabuki/courses/`) -}) - -const course = await response.json() -const lessons = course.lessonElements -let totalEpisodes = lessons.reduce((acc, cur) => typeof cur === 'number' ? acc += 1 : acc, 0) - -for (const data of Object.values(course.lessonData)) lessons[lessons.findIndex(x => x === data.index)] = { +for (const data of Object.values(course.lessonData)) course.lessonElements[course.lessonElements.findIndex(x => x === data.index)] = { title: data.title, slug: data.slug, url: `${data.sourceBase}/source?f=${PLAYLIST_EXT}`, index: data.index } -let j = 0; +const [lessons, totalEpisodes] = course.lessonElements.reduce((acc, cur) => { + if (typeof cur === 'string') (acc[0][cur] = [], acc[2] = cur) + else (acc[0][acc[2]].push(cur), acc[1]++) + return acc +}, [{}, 0, '']) -// \ / : * ? " < > | are not allowed in windows file name -const safePath = title => title.replace(/[\/\\:*?"<>|]/g, '') +let i = 1, x = totalEpisodes -for (let i = 0; i < lessons.length; i++) { - const episode = lessons[i] - - if (typeof episode === 'string') { - j = i - await ensureDir(join(safePath(course.title), safePath(lessons[j]))) - continue - } +const coursePath = safeJoin(DOWNLOAD_DIR, course.title) +for (const [lesson, episodes] of Object.entries(lessons)) { const - lessonName = lessons[j], - safeLessonName = safePath(lessonName), - safeCourseTitle = safePath(course.title), - safeEpisodeTitle = safePath(episode.title), - fileName = `${episode.index + 1}. ${safeEpisodeTitle}.${EXTENSION}`, - path = join(DOWNLOAD_DIR, safeCourseTitle, safeLessonName), - tempDir = join(path, '.tmp', safeEpisodeTitle), - filePath = join(tempDir, fileName), - decryptionKeyPath = join(tempDir, 'key.bin'), - captionPath = join(tempDir, `caption.${CAPTION_EXT}`), - playlistPath = join(tempDir, `playlist.${PLAYLIST_EXT}`), - finalFilePath = join(path, fileName) - - - spinner.text = `Downloading ${colors.red(lessonName)}/${colors.cyan().bold(fileName)} | Chunks: N/A | Remaining: ${--totalEpisodes}` - - if (await isPathExists(finalFilePath)) { - await sleep(100) - continue - } + lessonName = `${i++}. ${lesson}`, + lessonPath = safeJoin(coursePath, lessonName), + lessonTempDir = safeJoin(lessonPath, '.tmp') - await ensureDir(tempDir) + await ensureDir(lessonPath) - const - { url: m3u8Url } = await domFetch(episode.url, 'json'), - m3u8 = new M3u8(await domFetch([...m3u8Url.split('/').slice(0, -1), `${QUALITY}.${PLAYLIST_EXT}`].join('/'), 'text')), - key = await get(m3u8.decryptionKey), - captions = INCLUDE_CAPTION ? await get(`${FEM_CAPTIONS_ENDPOINT}/assets/courses/${course.datePublished}-${course.slug}/${episode.index}-${episode.slug}.${CAPTION_EXT}`) : null - - m3u8 - .setDecryptionKey('key.bin') + for (const episode of episodes) { + const + fileName = `${episode.index + 1}. ${episode.title}.${EXTENSION}`, + tempDir = safeJoin(lessonTempDir, QUALITY, episode.title), + decryptionKeyPath = safeJoin(tempDir, 'key.bin'), + captionPath = safeJoin(tempDir, `caption.${CAPTION_EXT}`), + playlistPath = safeJoin(tempDir, `playlist.${PLAYLIST_EXT}`), + filePath = safeJoin(tempDir, fileName), + finalFilePath = safeJoin(lessonPath, fileName) + + spinner.text = `Downloading ${colors.red(lessonName)}/${colors.cyan().bold(fileName)} | Chunks: N/A | Remaining: ${x--}/${totalEpisodes}` + + if (await isPathExists(finalFilePath)) { + await sleep(100) + continue + } - await Promise.all([ - fs.writeFile(decryptionKeyPath, key), - fs.writeFile(playlistPath, m3u8.toString()), - captions ? fs.writeFile(captionPath, captions) : Promise.resolve(), - ]) + await ensureDir(tempDir) - for (let x = 0; x < m3u8.totalChunks; x++) { const - chunkName = `${QUALITY}_${(x + 1).toString().padStart(5, '0')}.ts`, - chunkPath = join(tempDir, chunkName), - chunkUrl = [...m3u8Url.split('/').slice(0, -1), chunkName].join('/') + { url: m3u8Url } = await page.fetch(episode.url, 'json'), + m3u8 = new M3u8(await page.fetch([...m3u8Url.split('/').slice(0, -1), `${QUALITY}.${PLAYLIST_EXT}`].join('/'))), + key = await get(m3u8.decryptionKey), + captions = INCLUDE_CAPTION ? await get(`${FEM_CAPTIONS_ENDPOINT}/assets/courses/${course.datePublished}-${course.slug}/${episode.index}-${episode.slug}.${CAPTION_EXT}`) : null - if (await isPathExists(chunkPath)) { - continue - } + m3u8.setDecryptionKey('key.bin') - const chunk = await domFetch(chunkUrl, 'binary') - await fs.writeFile(chunkPath, chunk) - spinner.text = `Downloading ${colors.red(lessonName)}/${colors.cyan().bold(fileName)} | Chunks: ${x+1}/${m3u8.totalChunks} | Remaining: ${totalEpisodes}` - } + await Promise.all([ + fs.writeFile(decryptionKeyPath, key), + fs.writeFile(playlistPath, m3u8.toString()), + captions ? fs.writeFile(captionPath, captions) : Promise.resolve(), + ]) + + for (let j = 0; j < m3u8.totalChunks; j++) { + const + chunkName = `${QUALITY}_${(j + 1).toString().padStart(5, '0')}.ts`, + chunkPath = safeJoin(tempDir, chunkName), + chunkUrl = [...m3u8Url.split('/').slice(0, -1), chunkName].join('/') + + if (await isPathExists(chunkPath)) { + continue + } + + const chunk = await page.fetch(chunkUrl, 'binary') + + await fs.writeFile(chunkPath, chunk) + + spinner.text = `Downloading ${colors.red(lessonName)}/${colors.cyan().bold(fileName)} | Chunks: ${j + 1}/${m3u8.totalChunks} | Remaining: ${x + 1}/${totalEpisodes}` + } - // Merge chunks into one file. - await ffmpeg( - '-y', - '-allowed_extensions', 'ALL', - '-i', playlistPath, - '-map', '0', - '-c', - 'copy', INCLUDE_CAPTION ? filePath : finalFilePath - ) - - // Insert subtitles - if (INCLUDE_CAPTION) { - if (EXTENSION === 'mkv') { - await ffmpeg( - '-y', - '-i', filePath, - '-i', captionPath, - '-map', '0', - '-map', '1', - '-c', - 'copy', - finalFilePath - ) - } else { - await ffmpeg( - '-y', - '-i', filePath, - '-i', captionPath, - '-c', - 'copy', - '-c:s', 'mov_text', - '-metadata:s:s:0', 'language=eng', - finalFilePath - ) + // Merge chunks into one file. + await ffmpeg( + '-y', + '-allowed_extensions', 'ALL', + '-i', playlistPath, + '-map', '0', + '-c', + 'copy', INCLUDE_CAPTION ? filePath : finalFilePath + ) + + // Insert subtitles + if (INCLUDE_CAPTION) { + if (EXTENSION === 'mkv') { + await ffmpeg( + '-y', + '-i', filePath, + '-i', captionPath, + '-map', '0', + '-map', '1', + '-c', + 'copy', + finalFilePath + ) + } else { + await ffmpeg( + '-y', + '-i', filePath, + '-i', captionPath, + '-c', + 'copy', + '-c:s', 'mov_text', + '-metadata:s:s:0', 'language=eng', + finalFilePath + ) + } } } - await fs.rm(tempDir, { force: true, recursive: true }) + await fs.rm(lessonTempDir, { force: true, recursive: true }) } -spinner.text = 'Closing chrome engine... we almost done.' +spinner.text = 'Closing chromium engine... we almost done.' await browser.close() diff --git a/src/polyfill.js b/src/polyfill.js new file mode 100644 index 0000000..2048ed3 --- /dev/null +++ b/src/polyfill.js @@ -0,0 +1,19 @@ +import { Page } from 'puppeteer/lib/esm/puppeteer/common/Page.js' + +Page.prototype.fetch = async function (url, type = 'text') { + let code + + if (type === 'text' || type === 'json') code = `fetch("${url}", { credentials: "include" }).then(r => r.${type}())` + // https://github.com/puppeteer/puppeteer/issues/3722 + else if (type === 'binary') code = `fetch("${url}", { credentials: "include" }).then(r => new Promise(async resolve => { + const reader = new FileReader(); + reader.readAsBinaryString(await r.blob()); + reader.onload = () => resolve(reader.result); + reader.onerror = () => reject('Error occurred while reading binary string'); + }))` + else throw new Error('Unknown type: ' + type) + + const result = await this.evaluate(code) + + return type === 'binary' ? Buffer.from(result, 'binary') : result +} \ No newline at end of file diff --git a/src/util/common.js b/src/util/common.js index abbb9f1..af53ce0 100644 --- a/src/util/common.js +++ b/src/util/common.js @@ -1,5 +1,8 @@ import fs from 'node:fs/promises' import fetch from 'node-fetch' +import os from 'node:os' +import { join } from 'node:path' + export function formatBytes(bytes, decimals = 2) { if (!+bytes) return '0 Bytes' @@ -25,15 +28,22 @@ export async function ensureDir(path) { } export async function ensureEmpty(path) { - await fs.rm(path, { force: true, recursive: true }).catch(() => null) - await fs.mkdir(path, { recursive: true }) + await fs.rm(path, { force: true, recursive: true }).catch(() => null) + await fs.mkdir(path, { recursive: true }) } export function get(url) { - return fetch(url).then(res => { + return fetch(url).then((res) => { if (!res.ok) throw res return res.arrayBuffer() }).then(Buffer.from) } + +export function safeJoin(...path) { + path[path.length - 1] = os.platform() === 'win32' ? path[path.length - 1].replace(/[\/\\:*?"<>|]/g, '') : path[path.length - 1] + return join(...path) +} + + export { setTimeout as sleep } from 'node:timers/promises' \ No newline at end of file diff --git a/src/util/index.js b/src/util/index.js deleted file mode 100644 index 223b2f7..0000000 --- a/src/util/index.js +++ /dev/null @@ -1,3 +0,0 @@ -export * from './common.js' -export { default as ffmpeg } from './ffmpeg.js' -export * from './m3u8' \ No newline at end of file diff --git a/src/util/m3u8.js b/src/util/m3u8.js index f97ae85..a8475be 100644 --- a/src/util/m3u8.js +++ b/src/util/m3u8.js @@ -1,5 +1,5 @@ -export class M3u8 { +export default class M3u8 { constructor(content) { content = content.toString() @@ -7,9 +7,7 @@ export class M3u8 { this.decryptionKey = content.match(/URI="(.+)"/)[1] this.content = content.split('\n') - for (const line of this.content) { - if (!line.startsWith('#')) this.totalChunks++ - } + for (const line of this.content) if (line.startsWith('index_')) this.totalChunks++ } setDecryptionKey(key) {