diff --git a/README.md b/README.md index 66caa41..06a51d7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,22 @@ +## About This Fork + +This project is a small fork of the excellent [Obsidian Tweet to Markdown](https://github.com/kbravh/obsidian-tweet-to-markdown) project. This fork adds **Poll Tweets** functionality to automatically fetch new tweets from a Twitter timeline and add them to your Obsidian vault. + +The feature can be found at bottom of the existing plugin settings, under the "Additional Settings for Polling Tweets" heading. Enabling polling will periodically check for new tweets from the Twitter handles specified, and prepend them in reverse chronological order (just like appears in the normal Twitter user timeline) to a Markdown file named using the pattern specified. If the file does not exist, it will be created and *all* tweets added (up to the API limit which is 3200 at the time of writing). + +

+ Logo +

+ +**Note:** the current implementation is alpha "just-working" stage. It may not work with settings other than the defaults, errors may not be reported effectively and the code is rough. Improvements welcome! + +The README from the original project appears below. Please do not bug the original author with issues associated with the **Poll Tweets** feature in this fork. + +--- + +## Original Readme + +

diff --git a/images/poll-tweets-screenshot.png b/images/poll-tweets-screenshot.png new file mode 100644 index 0000000..0a2ba53 Binary files /dev/null and b/images/poll-tweets-screenshot.png differ diff --git a/main.ts b/main.ts index 9e647ae..fec2f2b 100644 --- a/main.ts +++ b/main.ts @@ -4,6 +4,7 @@ import {pasteTweet} from 'src/util' import {Tweet} from 'src/types/tweet' import {TweetCompleteModal} from 'src/TweetCompleteModal' import {TweetUrlModal} from 'src/TweetUrlModal' +import {createPollManager, PollManager} from 'src/pollManager' interface PasteFunction { (this: Plugin, ev: ClipboardEvent): void @@ -16,6 +17,8 @@ export default class TTM extends Plugin { bearerToken: string tweetComplete: TweetCompleteModal pasteFunction: PasteFunction + pollManager: PollManager + pollIntervalID: number //should be ReturnType but that seems to return the wrong type async onload(): Promise { console.info('Loading Tweet to Markdown') @@ -71,6 +74,8 @@ export default class TTM extends Plugin { }) this.addSettingTab(new TTMSettingTab(this.app, this)) + + this.pollManager = createPollManager(this.app, this) } onunload(): void { @@ -83,5 +88,24 @@ export default class TTM extends Plugin { async saveSettings(): Promise { await this.saveData(this.settings) + + if(this.settings.pollEnabled) { + if(this.pollIntervalID) + clearInterval(this.pollIntervalID) //registerInterval will still try to cancel it when unloading, but so be it. + + this.pollManager.pollRun(); + + var numHours = 24 //by default + switch(this.settings.pollFrequency) { + case 'monthly': { numHours = 24*30 } + case 'weekly': { numHours = 24*7 } + case 'twoDaily': { numHours = 24*2 } + case 'daily': { numHours = 24*1 } + case 'twiceDaily': { numHours = 24/2 } + case 'hourly': { numHours = 1 } + } + this.pollIntervalID = window.setInterval(() => this.pollManager.pollRun(), 1000*60*60*numHours) + this.registerInterval(this.pollIntervalID) + } } } diff --git a/src/pollManager.ts b/src/pollManager.ts new file mode 100644 index 0000000..47c542e --- /dev/null +++ b/src/pollManager.ts @@ -0,0 +1,150 @@ +import {App, Notice, TFile, parseFrontMatterEntry} from 'obsidian' +import TTM from 'main' +import {createDownloadManager, DownloadManager} from './downloadManager' +import {buildMarkdown, getBearerToken, getUser, getTimeline} from './util' +import {createFilename, doesFileExist, sanitizeFilename} from './util' +import type {Tweet, User} from './types/tweet' +import { TweetCompleteActions } from './types/plugin' + + +export const createPollManager = (app: App, plugin: TTM): PollManager => { + const pollRun = (): void => { + if (!navigator.onLine) { + new Notice('Polling for new tweets skipped because you appear to be offline.') + return + } + + const bearerToken = getBearerToken(plugin) + if (!bearerToken) { + new Notice('Polling for new tweets skipped because bearer token was not found.') + return + } + + const handles:string[] = plugin.settings.pollHandles.replace(/\s|@/g, '').split(",") + var downloadManager = createDownloadManager() + plugin.bearerToken = bearerToken + + handles.forEach(async handle => { + + let user + try { + //Note the term for a "handle" in the Twitter API is "username" + user = await getUser(handle, bearerToken); + } catch (error) { + new Notice(error.message) + return + } + + //Hella hacky way to make use of the existing createFilename interface + const dummyTweet:Tweet = { + includes: {users: [user]}, + data: {id: "", text: "", created_at: "", author_id: "", + public_metrics: {retweet_count: 0, reply_count: 0, like_count: 0, quote_count: 0}} + } + + let fname = createFilename( + dummyTweet, + plugin.settings.pollFilename, + { locale: plugin.settings.dateLocale, format: plugin.settings.dateFormat }) + + const fpath = createFilename( + dummyTweet, + plugin.settings.noteLocation ? plugin.settings.noteLocation : "./", //fix broken location default + { locale: plugin.settings.dateLocale, format: plugin.settings.dateFormat }, + 'directory') + + if (!fname || !fpath) { + new Notice('Failed to create filepath for timeline.') + return + } + + if (! await app.vault.adapter.exists(fpath)) { + await app.vault.createFolder(fpath).catch(error => { + new Notice('Error creating tweet directory.') + console.error( + 'There was an error creating the tweet directory.', + error + ) + return + }) + } + + //Seems getAbstractFileByPath doesn't handle "./" path. Strip it out if necessary here. + let abstractFile = app.vault.getAbstractFileByPath(fpath == "./" ? fname : `${fpath}/${fname}`) + let file: TFile + let since: Date + if(abstractFile && abstractFile instanceof TFile) { + file = abstractFile + if(file) + { + let metadata = app.metadataCache.getFileCache(file) + //If the parse fails, date will be the unix epoch, which is as good as undefined. + since = new Date(parseFrontMatterEntry(metadata.frontmatter, "fetched")) + } + } + + let tweets: Tweet[] + new Notice(`Polling for new Tweets from ${handle}...`) + + try { + tweets = await getTimeline(user.id, since, bearerToken) + } catch (error) { + new Notice(error.message) + return + } + + plugin.tweetMarkdown = '' + + const markdowns = await Promise.all( + tweets.map(async tweet => { + let markdown + try { + markdown = await buildMarkdown(app, plugin, downloadManager, tweet, 'normal', null) + } catch (error) { + new Notice('There was a problem processing the downloaded tweet') + console.error(error) + } + return markdown + }) + ) + + if(markdowns && markdowns.length > 0) { + plugin.tweetMarkdown = markdowns.join('\n\n---\n\n') + + await downloadManager + .finishDownloads() + .catch(error => { + new Notice('There was an error downloading the images.') + console.error(error) + }) + + // clean up excessive newlines + plugin.tweetMarkdown = plugin.tweetMarkdown.replace(/\n{2,}/g, '\n\n') + + // write the note to file + if(file) + { + var t = await app.vault.read(file) + app.vault.modify(file, [plugin.tweetMarkdown, t].join('\n\n---\n\n')) + new Notice(`${fname} updated.`) + } + else + { + const newFile = await app.vault.create(`${fpath}/${fname}`, plugin.tweetMarkdown) + new Notice(`${fname} created.`) + } + } + }) + + // clean up + plugin.tweetMarkdown = '' + } + + return { + pollRun, + } as PollManager +} + +export interface PollManager { + pollRun: () => void +} diff --git a/src/settings.ts b/src/settings.ts index 17021e4..6ef6fbe 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1,7 +1,8 @@ -import {App, moment, Platform, PluginSettingTab, Setting} from 'obsidian' +import {App, moment, Platform, PluginSettingTab, Setting, ToggleComponent} from 'obsidian' import {TweetCompleteAction, TweetCompleteActions} from './types/plugin' import {FolderSuggest} from './suggester/folderSuggester' import TTM from 'main' +import { exists } from 'fs' export interface TTMSettings { bearerToken: string | null @@ -29,6 +30,10 @@ export interface TTMSettings { includeDate: boolean dateFormat: string dateLocale: string + pollEnabled: boolean + pollHandles: string + pollFilename: string + pollFrequency: 'monthly' | 'weekly' | 'twoDaily' | 'daily' | 'twiceDaily' | 'hourly' } export const DEFAULT_SETTINGS: TTMSettings = { @@ -56,6 +61,10 @@ export const DEFAULT_SETTINGS: TTMSettings = { includeDate: true, dateFormat: 'LLL', dateLocale: 'en', + pollEnabled: false, + pollHandles: '', + pollFilename: 'Twitter timeline - [[handle]]', //Unfortunately, seems to displace the placeholder but how else do you set a default? + pollFrequency: 'daily' } export class TTMSettingTab extends PluginSettingTab { @@ -63,6 +72,7 @@ export class TTMSettingTab extends PluginSettingTab { locales = moment .locales() .reduce((obj, locale) => ({...obj, [locale]: locale}), {}) + pollEnableToggle: ToggleComponent constructor(app: App, plugin: TTM) { super(app, plugin) @@ -405,5 +415,77 @@ export class TTMSettingTab extends PluginSettingTab { await this.plugin.saveSettings() }) ) + + containerEl.createEl('h2', {text: 'Additional Settings for Polling Tweets'}) + + new Setting(containerEl) + .setName('Polling enabled') + .setDesc( + 'Periodically poll for new tweets.' + ) + .addToggle(toggle => { + toggle + .setValue(this.plugin.settings.pollEnabled) + .onChange(async value => { + this.plugin.settings.pollEnabled = value + await this.plugin.saveSettings() + }) + this.pollEnableToggle = toggle + }) + + new Setting(containerEl) + .setName('Twitter handles to poll') + .setDesc( + 'Handles (with or without @ prefix) to poll for new tweets, separated by commas.' + ) + .addText(text => + text + .setPlaceholder('@handle1,@handle2') + .setValue(this.plugin.settings.pollHandles) + .onChange(async value => { + this.plugin.settings.pollHandles = value + this.pollEnableToggle.setValue(false) + await this.plugin.saveSettings() + }) + ) + + new Setting(containerEl) + .setName('Filename to update with new tweets') + .setDesc( + 'The name of the files to add new tweets to. Files are created if they don\'t exist. You can use the placeholders [[handle]], [[name]], [[text]] and [[id]]. Defaults to "Twitter timeline - [[handle]]"' + ) + .addText(text => + text + .setPlaceholder('Twitter timeline - [[handle]]') + .setValue(this.plugin.settings.pollFilename) + .onChange(async value => { + this.plugin.settings.pollFilename = value + this.pollEnableToggle.setValue(false) + await this.plugin.saveSettings() + }) + ) + + new Setting(containerEl) + .setName('Poll frequency') + .setDesc( + 'How often to poll for new tweets.' + ) + .addDropdown(dropdown => + dropdown + .addOptions({ + monthly: 'Monthly', + weekly: 'Weekly', + twoDaily: 'Every second day', + daily: 'Every day', + twiceDaily: 'Twice daily', + hourly: 'Hourly' + }) + .setValue(this.plugin.settings.pollFrequency) + .onChange(async (value: 'monthly' | 'weekly' | 'twoDaily' | 'daily' | 'twiceDaily' | 'hourly' ) => { + this.plugin.settings.pollFrequency = value + this.pollEnableToggle.setValue(false) + await this.plugin.saveSettings() + }) + ) } } diff --git a/src/types/tweet.ts b/src/types/tweet.ts index 6a6e816..20f726d 100644 --- a/src/types/tweet.ts +++ b/src/types/tweet.ts @@ -119,3 +119,28 @@ export interface User { username: string profile_image_url: string } + +export interface UserResponse { + data: User + errors?: Error[] + // other error fields + reason?: string + status?: number +} + +export interface Meta { + result_count: number + newest_id: string + oldest_id: string + next_token?: string +} + +export interface Tweets { + includes: Includes + data: Data[] + meta: Meta + errors?: Error[] + // other error fields + reason?: string + status?: number +} diff --git a/src/util.ts b/src/util.ts index 2f86684..3eaecb7 100644 --- a/src/util.ts +++ b/src/util.ts @@ -19,6 +19,8 @@ import type { Tweet, TweetURL, User, + UserResponse, + Tweets, } from './types/tweet' import {decode} from 'html-entities' import {moment} from 'obsidian' @@ -849,3 +851,174 @@ export const getBearerToken = (plugin: TTM): string => { export const tweetStateCleanup = (plugin: TTM): void => { plugin.tweetMarkdown = '' } + + +export const getUser = async (username: string, bearer: string): Promise => { + if (bearer.startsWith('TTM')) { + return getUserFromTTM(username, bearer) + } + return getUserFromTwitter(username, bearer) +} + +/** + * Fetches a user object from the Twitter v2 API + * @param {string} username - The username of the user to fetch from the API + * @param {string} bearer - The bearer token + * @returns {User} - The user from the Twitter API + */ +const getUserFromTwitter = async ( + username: string, + bearer: string +): Promise => { + const twitterUrl = new URL(`https://api.twitter.com/2/users/by/username/${username}`) + const params = new URLSearchParams({ + 'user.fields': 'name,username,profile_image_url', + }) + + let userRequest + try { + userRequest = await request({ + method: 'GET', + url: `${twitterUrl.href}?${params.toString()}`, + headers: {Authorization: `Bearer ${bearer}`}, + }) + } catch (error) { + if (error.request) { + throw new Error('There seems to be a connection issue.') + } else { + console.error(error) + throw error + } + } + const userResponse: UserResponse = JSON.parse(userRequest) + if (userResponse.errors) { + throw new Error(userResponse.errors[0].detail) + } + if (userResponse?.status === 401) { + throw new Error('There seems to be a problem with your bearer token.') + } + if (userResponse?.reason) { + switch (userResponse.reason) { + case 'client-not-enrolled': + default: + throw new Error('There seems to be a problem with your bearer token.') + } + } + return userResponse.data +} + +/** + * Fetches a user object from the TTM service API + * @param {string} username - The username of the user to fetch from the API + * @param {string} bearer - The bearer token + * @returns {Promise} - The user from the Twitter API + */ +const getUserFromTTM = async (username: string, bearer: string): Promise => { + throw new Error('Fetching users from the TTM service API not currently supported.') + + return undefined +} + +export const getTimeline = async (userId: string, since: Date, bearer: string): Promise => { + if (bearer.startsWith('TTM')) { + return getTimelineFromTTM(userId, since, bearer) + } + return getTimelineFromTwitter(userId, since, bearer) +} + +/** + * Fetches a user's tweet timeline as an array of tweets in reverse chronological order from the Twitter v2 API + * @param {string} userId - The userId of the timeline to fetch from the API + * @param {Date} since - Fetch all tweets after this date. Pass "undefined" to fetch all tweets. + * @param {string} bearer - The bearer token + * @returns {Tweet[]} - The tweets from the Twitter API + */ +const getTimelineFromTwitter = async (userId: string, since: Date, bearer: string): Promise => { + const twitterUrl = new URL(`https://api.twitter.com/2/users/${userId}/tweets`) + const params = new URLSearchParams({ + expansions: 'author_id,attachments.poll_ids,attachments.media_keys', + 'user.fields': 'name,username,profile_image_url', + 'tweet.fields': + 'attachments,public_metrics,entities,conversation_id,referenced_tweets,created_at', + 'media.fields': 'url,alt_text', + 'poll.fields': 'options', + }) + + if(since) + params.set("start_time", since.toISOString()); + + var hasNextPage:boolean = true + var nextToken:string = undefined + var ret:Tweet[] = [] + + while (hasNextPage) { + + let tweetsRequest + try { + if(nextToken) { + params.set("pagination_token", nextToken) + } + + tweetsRequest = await request({ + method: 'GET', + url: `${twitterUrl.href}?${params.toString()}`, + headers: {Authorization: `Bearer ${bearer}`}, + }) + } catch (error) { + if (error.request) { + throw new Error('There seems to be a connection issue.') + } else { + console.error(error) + throw error + } + } + const tweets: Tweets = JSON.parse(tweetsRequest) + if (tweets.errors) { + throw new Error(tweets.errors[0].detail) + } + if (tweets?.status === 401) { + throw new Error('There seems to be a problem with your bearer token.') + } + if (tweets?.reason) { + switch (tweets.reason) { + case 'client-not-enrolled': + default: + throw new Error('There seems to be a problem with your bearer token.') + } + } + + if(tweets.data) { + tweets.data.forEach(t => { + var media = tweets.includes.media && t.attachments && t.attachments.media_keys ? + tweets.includes.media.filter(m => t.attachments.media_keys.includes(m.media_key)) : + null + var polls = tweets.includes.polls && t.attachments && t.attachments.poll_ids ? + tweets.includes.polls.filter(p => t.attachments.poll_ids.includes(p.id)) : + null + ret.push({ + includes: {users: tweets.includes.users, media: media, polls: polls}, + data: t + }) + }) + } + + if (tweets.meta.next_token) { + nextToken = tweets.meta.next_token; + } else { + hasNextPage = false; + } + } + + return ret +} + +/** + * Fetches a user object from the TTM service API + * @param {string} userId - The userId of the timeline to fetch from the API + * @param {Date} since - Fetch all tweets after this date. Pass "undefined" to fetch all tweets. + * @param {string} bearer - The bearer token + * @returns {Promise} - The user from the Twitter API + */ +const getTimelineFromTTM = async (userId: string, since: Date, bearer: string): Promise => { + throw new Error('Fetching users from the TTM service API not currently supported.') +}