Merge pull request #10 from breezykermo/topic/abstract-clients

Python and DevonTHINK clients, Docker, and ability to clip with a note

.gitignore

@@ -0,0 +1,2 @@
+clips.json
+assets/

Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3.9-slim-buster
+RUN mkdir -p /app
+WORKDIR /app
+ADD server.py /app/server.py

clients/devonthink/HiliClip.applescript

@@ -0,0 +1,53 @@
+(*
+    Clip contents of existing selection to a hili server.
+    See https://github.com/breezykermo/hili for more info.
+*)
+on replace_chars(this_text, search_string, replacement_string)
+  set AppleScript's text item delimiters to the search_string
+  set the item_list to every text item of this_text
+  set AppleScript's text item delimiters to the replacement_string
+  set this_text to the item_list as string
+  set AppleScript's text item delimiters to ""
+  return this_text
+end replace_chars
+
+tell application id "DNtp"
+  try
+    set theDoc to the content record of think window 1
+    set theTitle to ((the name without extension of theDoc) as string)
+    set theRefURL to the reference URL of theDoc as string
+    set thePage to ((the current page of think window 1) as string)
+    set theUrl to theRefURL & "?page=" & (thePage as string)
+
+    set theCitedText to the (selected text of think window 1 as string)
+    if theCitedText is "" then
+      (* Scenario 2: Document is open, but no text is highlighted. *)
+      set theQuotedText to missing value
+    else
+      (* Scenario 3: Document is open, text is highlighted. *)
+      set theQuotedText to my theCitedText
+    end if
+
+    -- set theTags to the tags of theDoc
+    set _note to display dialog "note" default answer "" buttons {"Cancel", "Continue"} default button "Continue"
+    set _tags to display dialog "tags" default answer "" buttons {"Cancel", "Continue"} default button "Continue"
+    set theNote to the text returned of _note
+    set theTags to the text returned of _tags
+    set theQuote to my replace_chars(theQuotedText, "\\n", "\\\\n")
+
+    do shell script "touch /tmp/args.txt"
+    do shell script "echo " & quoted form of theTitle & " > /tmp/args.txt"
+    do shell script "echo '*--STARTQUOTE--*' >> /tmp/args.txt"
+    do shell script "echo " & quoted form of theQuote & " >> /tmp/args.txt"
+    do shell script "echo '*--ENDQUOTE--*' >> /tmp/args.txt"
+    do shell script "echo " & quoted form of theNote & " >> /tmp/args.txt"
+    do shell script "echo " & quoted form of theTags & " >> /tmp/args.txt"
+    do shell script "echo " & quoted form of theUrl & " >> /tmp/args.txt"
+
+    (* NOTE: change the following line to suit your system *)
+    do shell script "cd /absolute/path/to/hili/clients/python && URL='http://localhost:8888' python3 clip.py > /tmp/hili_clip_log.txt"
+
+	on error error_message number error_number
+		if the error_number is not -128 then display alert "DEVONthink Pro" message error_message as warning
+	end try
+end tell

extension/hili.js → clients/firefox/hili.js

@@ -128,16 +128,18 @@ function highlightText() {
     html = getSelectionHtml(selection);
   }
   if (text) {
+    let note = prompt('Note', '').trim()
     let tags = prompt('Tags', last_tags).split(',').map((t) => t.trim());
     if (tags == null) return;
     last_tags = tags.join(', ');
     let data = {
       href: cleanUrl(window.location.href),
       title: document.title,
       time: +new Date(),
-      text: text,
-      html: html,
-      tags: tags
+      text,
+      html,
+      note,
+      tags,
     };
     post(data);
   }
@@ -182,7 +184,7 @@ const marketingRegex =  /(utm_.+|mc_.+|cmpid|truid|CMP)/;
 function cleanUrl(url) {
   url = new URL(url);
   let toDelete = new Set();
-  for (let entry of url.searchParams) { 
+  for (let entry of url.searchParams) {
     let [key, val] = entry;
     if (marketingRegex.test(key)) {
       toDelete.add(key);

clients/python/clip.py

@@ -0,0 +1,96 @@
+import sys
+import os
+import json
+import requests
+import time
+
+# cheap args
+env_args = os.environ.get("ARGS")
+env_url = os.environ.get("URL")
+env_pw = os.environ.get("PASSWORD")
+
+if env_url is None:
+    sys.exit("You must specify a server 'URL' in the environment.")
+
+# globals
+ARGS = env_args if env_args is not None else "/tmp/args.txt"
+SERVER_URL = env_url
+PASSWORD = env_pw
+CACHE = "./cached_clips.json" # not in temp so it isn't removed
+
+
+def send(body):
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+    if PASSWORD is not None:
+        headers["Authentication"] = PASSWORD
+
+    requests.post(
+        SERVER_URL,
+        headers = headers,
+        json = body
+    )
+
+
+def attempt_clip(clip):
+    try:
+        send(clip)
+
+        # if clip is successful, flush all cached
+        if os.path.exists(CACHE):
+            with open(CACHE, "r") as c:
+                cached_clips = [json.loads(l) for l in c.readlines()]
+
+            for cached_clip in cached_clips:
+                send(cached_clip)
+
+            os.remove(CACHE)
+
+    # TODO: only catch the specifics
+    except requests.ConnectionError:
+        is_first = not os.path.exists(CACHE)
+        with open(CACHE, "a") as cache:
+            if not is_first: cache.write("\n")
+            json.dump(clip, cache)
+        print("No internet connection, dumped to cache")
+
+
+def run():
+    with open(ARGS, 'r') as f:
+        data = f.readlines()
+    tm = int(round(time.time() * 1000))
+
+    idx = 0
+    title = ""
+    while data[idx] != "*--STARTQUOTE--*\n" and idx < len(data):
+        title += data[idx]
+        idx += 1
+
+    idx += 1 # skip STARTQUOTE
+    quote = ""
+    while data[idx] != "*--ENDQUOTE--*\n" and idx < len(data):
+        quote += data[idx]
+        idx += 1
+    idx += 1 # skip ENDQUOTE
+
+    note = data[idx].rstrip("\n").strip()
+    tags = data[idx + 1].rstrip("\n").strip().split(",")
+    if len(tags) == 1 and tags[0] == "": tags = []
+    url = data[idx + 2].rstrip("\n").strip()
+
+    clip = {
+        "time": tm,
+        "title": title,
+        "html": quote,
+        "note": note,
+        "tags": tags,
+        "href": url,
+    }
+
+    attempt_clip(clip)
+
+
+if __name__ == "__main__":
+    run()

clients/python/example.txt

@@ -0,0 +1,7 @@
+Title of the document
+*--STARTQUOTE--*
+This is the quote you want to send to hili, usually drawn from whatever is highlighted in an application at the time of clipping.
+*--ENDQUOTE--*
+Here is the note that you've written, normally entered through a prompt at clip time.
+test,example
+https://lachlankermode.com/example-url-for-hili

clients/python/readme.md

@@ -0,0 +1,7 @@
+Simple python script to send a clip to a hili server.
+
+## Run
+```bash
+URL=http://localhost:8888 ARGS=$(pwd)/example.txt python clip.py
+```
+

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+services:
+  hili:
+    build: .
+    command: python server.py /app/clips.json /app/assets
+    volumes:
+      - ./clips.json:/app/clips.json
+      - ./assets:/app/assets
+    ports:
+      - "8888:8888"
+
+

readme.md

@@ -1,18 +1,30 @@
 # hili
 
-Firefox extension to highlight and save text and images from the web.
+Barebones server that listens for clips to save text and images from the web.
+Includes a set of clients to send clips from various apps:
 
-If you are offline, `hili` queues the snippets to send when a connection returns (you need to leave the tab open until they are successfully sent; there's a little indicator that lets you know how many snippets are queued).
+- Firefox extension
+- iOS (via [Scriptable](https://scriptable.app/))
+- [DevonTHINK](https://www.devontechnologies.com/)
+- Python
 
-## Extension
+Firefox is the best supported client. If you are offline, `hili` queues the
+snippets to send when a connection returns (you need to leave the tab open
+until they are successfully sent; there's a little indicator that lets you know
+how many snippets are queued).
 
-To install (Firefox), open `about:debugging` and choose "Load Temporary Add-on", then select the `manifest.json` file. This is temporary (but useful for development); the add-on will be gone next time you run Firefox.
+## Firefox
+
+To install (Firefox), open `about:debugging` and choose "Load Temporary
+Add-on", then select the `manifest.json` file. This is temporary (but useful
+for development); the add-on will be gone next time you run Firefox.
 
 To install it more permanently:
 
 If you're running Firefox Developer Edition, you should be able to:
 1. Zip up the `extensions` directory
-2. Go to `about:addons`, then `Install Add-on From File`, and select the zipped extension
+2. Go to `about:addons`, then `Install Add-on From File`, and select the zipped
+   extension
 
 Otherwise, the process is more involved:
 1. Go to `https://addons.mozilla.org/en-US/developers/addon/api/key/` (create a Firefox account if necessary) and generate credentials

server.py

@@ -17,8 +17,22 @@
 parser.add_argument('UPLOAD_DIR', type=str, help='Directory to save uploaded files')
 parser.add_argument('-p', '--port', type=int, dest='PORT', default=8888, help='Port for server')
 parser.add_argument('-k', '--key', type=str, dest='KEY', default=None, help='Secret key to authenticate clients')
+parser.add_argument('-o', '--key-overrides', type=str, dest='OVERRIDES', default=None, help='Key overrides for clips sent to server in the format "original1:new1;original2:new2"')
 args = parser.parse_args()
 
+def idx(obj, base, overrides):
+    key = base
+    for override in overrides:
+        a,b = override.split(":")
+        if a == base:
+            key = b
+    return obj.get(key) if obj.get(key) is not None else obj.get(base)
+
+overrides = []
+if args.OVERRIDES:
+    overrides = args.OVERRIDES.split(";")
+    overrides = [x for x in overrides if len(x.split(":")) == 2]
+
 
 class JSONRequestHandler(BaseHTTPRequestHandler):
     def do_POST(self):
@@ -99,6 +113,11 @@ def do_GET(self):
                         .highlight {
                             margin: 2em 0;
                         }
+                        .note {
+                            margin-top: 0.5em;
+                            text-align: right;
+                            font-size: 0.9em;
+                        }
                         .tags {
                             color: #888;
                             margin-top: 1em;
@@ -116,36 +135,36 @@ def do_GET(self):
 
         grouped = defaultdict(list)
         for d in data:
-            grouped[d['href']].append(d)
+            grouped[idx(d, 'href', overrides)].append(d)
 
-        for href, group in sorted(grouped.items(), key=lambda g: -max([d['time'] for d in g[1]])):
+        for href, group in sorted(grouped.items(), key=lambda g: -max([idx(d, 'time', overrides) for d in g[1]])):
             html.append('''
                 <article>
-                    <h4><a href="{href}">{title}</a></h4>'''.format(href=href, title=group[0]['title']))
+                    <h4><a href="{href}">{title}</a></h4>'''.format(href=href, title=group[0].get('title')))
             for d in group:
                 if 'file' in d:
-                    # fname = d['file']['name']
                     html.append('''
                         <div class="highlight">
                             <img src="{src}">
                             <p>{text}</p>
                             <div class="tags"><em>{tags}</em></div>
                         </div>
                     '''.format(
-                        # src=os.path.join(args.UPLOAD_DIR, fname),
-                        src=d['file']['src'],
-                        text=d['text'],
-                        tags=', '.join(d['tags'])
+                        src=idx(d, 'file', overrides)['src'],
+                        text=idx(d, 'text', overrides),
+                        tags=', '.join(idx(d, 'tags', overrides))
                     ))
                 else:
                     html.append('''
                         <div class="highlight">
                             {html}
+                            <div class="note">{note}</div>
                             <div class="tags"><em>{tags}</em></div>
                         </div>
                     '''.format(
-                        html=d['html'],
-                        tags=', '.join(d['tags'])
+                        html=idx(d, 'html', overrides),
+                        note=idx(d, 'note', overrides),
+                        tags=', '.join(idx(d, 'tags', overrides))
                     ))
             html.append('</article>')