PCC-1026 Reorganize importer code. Implement smart component replacement for drupal import.

packages/cli/package.json

@@ -49,6 +49,7 @@
     "google-auth-library": "^9.4.0",
     "googleapis": "^129.0.0",
     "inquirer": "^8.2.6",
+    "node-html-parser": "^6.1.12",
     "nunjucks": "^3.2.4",
     "octokit": "^3.1.2",
     "open": "^9.1.0",

packages/cli/src/cli/commands/import.ts → packages/cli/src/cli/commands/import/drupal.ts

@@ -1,22 +1,17 @@
 import { randomUUID } from "crypto";
-import * as fs from "fs";
 import { exit } from "process";
 import axios, { AxiosError } from "axios";
 import Promise from "bluebird";
 import chalk from "chalk";
-import { parseFromString } from "dom-parser";
 import type { GaxiosResponse } from "gaxios";
 import { OAuth2Client } from "google-auth-library";
 import { drive_v3, google } from "googleapis";
-import ora from "ora";
+import { HTMLElement, parse } from "node-html-parser";
 import queryString from "query-string";
-import showdown from "showdown";
-import AddOnApiHelper from "../../lib/addonApiHelper";
-import { getLocalAuthDetails } from "../../lib/localStorage";
-import { Logger } from "../../lib/logger";
-import { errorHandler } from "../exceptions";
-
-const HEADING_TAGS = ["h1", "h2", "h3", "title"];
+import { getLocalAuthDetails } from "../../../lib/localStorage";
+import { Logger } from "../../../lib/logger";
+import { errorHandler } from "../../exceptions";
+import AddOnApiHelper from "../../../lib/addonApiHelper";
 
 type DrupalImportParams = {
   baseUrl: string;
@@ -182,7 +177,9 @@
           (x) => x.id === post.relationships.field_author.data.id,
         )?.attributes?.title;
 
-        const res = (await drive.files.create({
+        // Initially create a blank document, just to get an article id
+        // that we can work with for further steps, such as adding smart components.
+        let res = (await drive.files.create({
           requestBody: {
             // Name from the article.
             name: post.attributes.title,
@@ -191,7 +188,7 @@
           },
           media: {
             mimeType: "text/html",
-            body: post.attributes.body.processed,
+            body: "",
           },
         })) as GaxiosResponse<drive_v3.Schema$File>;
         const fileId = res.data.id;
@@ -203,6 +200,21 @@
         // Add it to the PCC site.
         await AddOnApiHelper.getDocument(fileId, true);
 
+        // Set the document's content.
+        res = (await drive.files.update({
+          requestBody: {
+            id: fileId,
+            mimeType: "application/vnd.google-apps.document",
+          },
+          media: {
+            mimeType: "text/html",
+            body: await processHTMLForSmartComponents(
+              post.attributes.body.processed,
+              fileId,
+            ),
+          },
+        })) as GaxiosResponse<drive_v3.Schema$File>;
+
         try {
           await AddOnApiHelper.updateDocument(
             fileId,
@@ -241,104 +253,38 @@
   },
 );
 
-type MarkdownImportParams = {
-  filePath: string;
-  siteId: string;
-  verbose: boolean;
-  publish: boolean;
-};
-
-export const importFromMarkdown = errorHandler<MarkdownImportParams>(
-  async ({ filePath, siteId, verbose, publish }: MarkdownImportParams) => {
-    const logger = new Logger();
+async function processHTMLForSmartComponents(html: string, articleId: string) {
+  const root = parse(html);
+  const iframeNodes: HTMLElement[] =
+    (root.querySelector("iframe")?.childNodes as HTMLElement[]) ?? [];
 
-    if (!fs.existsSync(filePath)) {
-      logger.error(
-        chalk.red(
-          `ERROR: Could not find markdown file at given path (${filePath})`,
-        ),
-      );
-      exit(1);
-    }
+  await Promise.all(
+    iframeNodes.map(async (node) => {
+      let src = node.getAttribute("src");
 
-    // Prepare article content and title
-    const content = fs.readFileSync(filePath).toString();
-
-    // Check user has required permission to create drive file
-    await AddOnApiHelper.getIdToken([
-      "https://www.googleapis.com/auth/drive.file",
-    ]);
-    const authDetails = await getLocalAuthDetails();
-    if (!authDetails) {
-      logger.error(chalk.red(`ERROR: Failed to retrieve login details.`));
-      exit(1);
-    }
+      if (src == null) return;
 
-    // Create Google Doc
-    const spinner = ora("Creating document on the Google Drive...").start();
-    const oauth2Client = new OAuth2Client();
-    oauth2Client.setCredentials(authDetails);
-    const drive = google.drive({
-      version: "v3",
-      auth: oauth2Client,
-    });
-    const converter = new showdown.Converter();
-    const html = converter.makeHtml(content);
-    const dom = parseFromString(html);
-
-    // Derive document's title
-    let title: string | undefined = undefined;
-    for (const item of HEADING_TAGS) {
-      const element = dom.getElementsByTagName(item)[0];
-      if (element) {
-        title = element.textContent;
-        break;
+      if (src.includes("oembed?url=")) {
+        src = decodeURIComponent(src.split("oembed?url=")[1]);
       }
-    }
-    title = title || "Untitled Document";
-
-    const res = (await drive.files.create({
-      requestBody: {
-        name: title,
-        mimeType: "application/vnd.google-apps.document",
-      },
-      media: {
-        mimeType: "text/html",
-        body: html,
-      },
-    })) as GaxiosResponse<drive_v3.Schema$File>;
-    const fileId = res.data.id;
-    const fileUrl = `https://docs.google.com/document/d/${fileId}`;
 
-    if (!fileId) {
-      spinner.fail("Failed to create document on the Google Drive.");
-      exit(1);
-    }
+      const componentType = "MEDIA_PREVIEW";
+      const componentId = await AddOnApiHelper.createSmartComponent(
+        articleId,
+        {
+          url: src,
+          canUsePlainIframe: true,
+        },
+        componentType,
+      );
 
-    // Create PCC document
-    await AddOnApiHelper.getDocument(fileId, true, title);
-    // Cannot set metadataFields(title,slug) in the same request since we reset metadataFields
-    //  when changing the siteId.
-    await AddOnApiHelper.updateDocument(
-      fileId,
-      siteId,
-      title,
-      [],
-      null,
-      verbose,
-    );
-    await AddOnApiHelper.getDocument(fileId, false, title);
+      node.replaceWith(
+        parse(
+          `<a href="https://pcc.media/${componentType}/${componentId}">MEDIA_PREVIEW: ${src}</a>`,
+        ),
+      );
+    }),
+  );
 
-    // Publish PCC document
-    if (publish) {
-      const { token } = await oauth2Client.getAccessToken();
-      await AddOnApiHelper.publishDocument(fileId, token as string);
-    }
-    spinner.succeed(
-      `Successfully created document at below path${
-        publish ? " and published it on the PCC." : ":"
-      }`,
-    );
-    logger.log(chalk.green(fileUrl, "\n"));
-  },
-);
+  return root.toString();
+}

packages/cli/src/cli/commands/import/index.ts

@@ -0,0 +1,4 @@
+import { importFromDrupal } from "./drupal";
+import { importFromMarkdown } from "./markdown";
+
+export { importFromDrupal, importFromMarkdown };

packages/cli/src/cli/commands/import/markdown.ts

@@ -0,0 +1,117 @@
+import * as fs from "fs";
+import { exit } from "process";
+import chalk from "chalk";
+import { parseFromString } from "dom-parser";
+import type { GaxiosResponse } from "gaxios";
+import { OAuth2Client } from "google-auth-library";
+import { drive_v3, google } from "googleapis";
+import ora from "ora";
+import showdown from "showdown";
+import AddOnApiHelper from "../../../lib/addonApiHelper";
+import { getLocalAuthDetails } from "../../../lib/localStorage";
+import { Logger } from "../../../lib/logger";
+import { errorHandler } from "../../exceptions";
+
+const HEADING_TAGS = ["h1", "h2", "h3", "title"];
+
+type MarkdownImportParams = {
+  filePath: string;
+  siteId: string;
+  verbose: boolean;
+  publish: boolean;
+};
+
+export const importFromMarkdown = errorHandler<MarkdownImportParams>(
+  async ({ filePath, siteId, verbose, publish }: MarkdownImportParams) => {
+    const logger = new Logger();
+
+    if (!fs.existsSync(filePath)) {
+      logger.error(
+        chalk.red(
+          `ERROR: Could not find markdown file at given path (${filePath})`,
+        ),
+      );
+      exit(1);
+    }
+
+    // Prepare article content and title
+    const content = fs.readFileSync(filePath).toString();
+
+    // Check user has required permission to create drive file
+    await AddOnApiHelper.getIdToken([
+      "https://www.googleapis.com/auth/drive.file",
+    ]);
+    const authDetails = await getLocalAuthDetails();
+    if (!authDetails) {
+      logger.error(chalk.red(`ERROR: Failed to retrieve login details.`));
+      exit(1);
+    }
+
+    // Create Google Doc
+    const spinner = ora("Creating document on the Google Drive...").start();
+    const oauth2Client = new OAuth2Client();
+    oauth2Client.setCredentials(authDetails);
+    const drive = google.drive({
+      version: "v3",
+      auth: oauth2Client,
+    });
+    const converter = new showdown.Converter();
+    const html = converter.makeHtml(content);
+    const dom = parseFromString(html);
+
+    // Derive document's title
+    let title: string | undefined = undefined;
+    for (const item of HEADING_TAGS) {
+      const element = dom.getElementsByTagName(item)[0];
+      if (element) {
+        title = element.textContent;
+        break;
+      }
+    }
+    title = title || "Untitled Document";
+
+    const res = (await drive.files.create({
+      requestBody: {
+        name: title,
+        mimeType: "application/vnd.google-apps.document",
+      },
+      media: {
+        mimeType: "text/html",
+        body: html,
+      },
+    })) as GaxiosResponse<drive_v3.Schema$File>;
+    const fileId = res.data.id;
+    const fileUrl = `https://docs.google.com/document/d/${fileId}`;
+
+    if (!fileId) {
+      spinner.fail("Failed to create document on the Google Drive.");
+      exit(1);
+    }
+
+    // Create PCC document
+    await AddOnApiHelper.getDocument(fileId, true, title);
+    // Cannot set metadataFields(title,slug) in the same request since we reset metadataFields
+    //  when changing the siteId.
+    await AddOnApiHelper.updateDocument(
+      fileId,
+      siteId,
+      title,
+      [],
+      null,
+      verbose,
+    );
+    await AddOnApiHelper.getDocument(fileId, false, title);
+
+    // Publish PCC document
+    if (publish) {
+      const { token } = await oauth2Client.getAccessToken();
+      await AddOnApiHelper.publishDocument(fileId, token as string);
+    }
+    spinner.succeed(
+      `Successfully created document at below path${
+        publish ? " and published it on the PCC." : ":"
+      }`,
+    );
+    logger.log(chalk.green(fileUrl, "\n"));
+  },
+);

packages/cli/src/lib/addonApiHelper.ts

@@ -1,3 +1,4 @@
+import { SmartComponentMap } from "@pantheon-systems/pcc-sdk-core/types";
 import axios, { AxiosError, HttpStatusCode } from "axios";
 import { Credentials } from "google-auth-library";
 import ora from "ora";
@@ -226,6 +227,40 @@
     }
   }
 
+  static async createSmartComponent(
+    articleId: string,
+    attributes: { [key: string]: string | number | boolean | null | undefined },
+    componentType: string,
+  ): Promise<string> {
+    const idToken = await this.getIdToken();
+
+    try {
+      return (
+        await axios.post(
+          `${API_KEY_ENDPOINT}/components`,
+          {
+            articleId,
+            attributes,
+            componentType,
+          },
+          {
+            headers: {
+              Authorization: `Bearer ${idToken}`,
+            },
+          },
+        )
+      ).data.id;
+    } catch (err) {
+      if (
+        (err as { response: { status: number } }).response.status ===
+        HttpStatusCode.NotFound
+      )
+        throw new HTTPNotFound();
+
+      throw err;
+    }
+  }
+
   static async createSite(url: string): Promise<string> {
     const idToken = await this.getIdToken();