Add extract command (#21)

* add extract command

* small rfks, finish cmd

* bump ver

README.md

@@ -39,7 +39,8 @@ Commands:
   dria pull [contract]          Pull a knowledge to your local machine.
   dria serve [contract]         Serve a local knowledge.
   dria clear [contract]         Clear local knowledge.
-  dria fetch <txid>             Fetch an existing index at the given URL directly.
+  dria fetch <txid>             Fetch an existing index on Arweave.
+  dria extract <zip-path>       Extract a compressed knowledge.
   dria set-contract <contract>  Set default contract.
   dria config                   Show default configurations.
   dria list                     List all local knowledge.
@@ -93,6 +94,14 @@ dria fetch <txid>
 
 Note that the argument here is not the knowledge ID (i.e. the corresponding Arweave contract txID); instead, it is the transaction ID of the bundling transaction where the RocksDB folder was zipped & uploaded to Arweave.
 
+### Extract Knowledge
+
+The previously mentioned [fetch](#fetch-knowledge) command downloads a zip file from Arweave & extracts it. However, in some cases you might have the knowledge on another platform that is easier to access compared to downloading from Arweave, especially if internet connection is an issue. For such cases, we have the extract command which simply unzips the compressed knowledge under Dria's data directory.
+
+```sh
+dria extract <zip-path>
+```
+
 ### Configurations
 
 You can set the default contract with `set-contract` command. When a contract is set by default, `[contract]` can be omitted such that the CLI will use the default one. To see the defaults:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dria-cli",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "description": "A command-line tool for Dria",
   "author": "FirstBatch Team <[email protected]>",
   "contributors": [

src/cli/args.ts

@@ -0,0 +1,57 @@
+import { logger } from "../common";
+import { getConfig } from "../configurations";
+
+const config = getConfig();
+
+/** ContractID parameter, as seen on every Dria knowledge. */
+const contractId = {
+  id: "contract" as const,
+  opts: {
+    alias: "c",
+    describe: "Contract ID",
+    type: "string",
+    default: config.contract,
+  } as const,
+} as const;
+
+/** Verbosity, will show extra information when enabled. */
+const verbose = {
+  id: "verbose" as const,
+  opts: {
+    alias: "v",
+    describe: "Show extra information",
+    boolean: true,
+    default: false,
+    coerce: (verbose: boolean) => {
+      logger.setLevel(verbose ? "DEBUG" : "INFO");
+      return verbose;
+    },
+  } as const,
+} as const;
+
+/** An Arweave txID, required for the `fetch` command. */
+const txId = {
+  id: "txid" as const,
+  opts: {
+    describe: "Transaction ID",
+    type: "string",
+    demandOption: true,
+  } as const,
+} as const;
+
+/** A zip path, required for the `extract` command. */
+const zipPath = {
+  id: "zipPath" as const,
+  opts: {
+    describe: "Path to zip",
+    type: "string",
+    demandOption: true,
+  } as const,
+} as const;
+
+export default {
+  txId,
+  zipPath,
+  contractId,
+  verbose,
+} as const;

src/cli.ts → src/cli/index.ts

@@ -1,42 +1,8 @@
 import yargs from "yargs";
-import commands from "./commands/";
-import { checkDocker, checkNetwork, logger } from "./common";
-import { getConfig, setConfig } from "./configurations";
-
-const config = getConfig();
-
-const contractIdArg = {
-  id: "contract" as const,
-  opts: {
-    alias: "c",
-    describe: "Contract ID",
-    type: "string",
-    default: config.contract,
-  } as const,
-} as const;
-
-const verboseArg = {
-  id: "verbose" as const,
-  opts: {
-    alias: "v",
-    describe: "Show extra information",
-    boolean: true,
-    default: false,
-    coerce: (verbose: boolean) => {
-      logger.setLevel(verbose ? "DEBUG" : "INFO");
-      return verbose;
-    },
-  } as const,
-} as const;
-
-const txIdArg = {
-  id: "txid" as const,
-  opts: {
-    describe: "Transaction ID",
-    type: "string",
-    demandOption: true,
-  } as const,
-} as const;
+import commands from "../commands/";
+import args from "./args";
+import { checkDocker, checkNetwork, logger } from "../common";
+import { getConfig, setConfig } from "../configurations";
 
 async function checkArgs(args: { contract?: string }, checks: { contract?: boolean; docker?: boolean }) {
   if (checks.contract) {
@@ -59,16 +25,16 @@ async function checkArgs(args: { contract?: string }, checks: { contract?: boole
  *
  * driaCLI(hideBin(process.argv));
  */
-export function driaCLI(args: string[]) {
-  yargs(args)
+export function driaCLI(cliArgs: string[]) {
+  yargs(cliArgs)
     .scriptName("dria")
-    .option(verboseArg.id, verboseArg.opts)
+    .option(args.verbose.id, args.verbose.opts)
 
     .command(
       "pull [contract]",
       "Pull a knowledge to your local machine.",
       (yargs) =>
-        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+        yargs.positional(args.contractId.id, args.contractId.opts).check(async (args) => {
           return await checkArgs(args, { contract: true, docker: true });
         }),
       async (args) => {
@@ -80,7 +46,7 @@ export function driaCLI(args: string[]) {
       "serve [contract]",
       "Serve a local knowledge.",
       (yargs) =>
-        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+        yargs.positional(args.contractId.id, args.contractId.opts).check(async (args) => {
           return await checkArgs(args, { contract: true, docker: true });
         }),
       async (args) => {
@@ -92,7 +58,7 @@ export function driaCLI(args: string[]) {
       "clear [contract]",
       "Clear local knowledge.",
       (yargs) =>
-        yargs.positional(contractIdArg.id, contractIdArg.opts).check(async (args) => {
+        yargs.positional(args.contractId.id, args.contractId.opts).check(async (args) => {
           return await checkArgs(args, { contract: true });
         }),
       async (args) => {
@@ -102,17 +68,26 @@ export function driaCLI(args: string[]) {
 
     .command(
       "fetch <txid>",
-      "Fetch an existing index at the given URL directly.",
-      (yargs) => yargs.positional(txIdArg.id, txIdArg.opts),
+      "Fetch an existing index on Arweave.",
+      (yargs) => yargs.positional(args.txId.id, args.txId.opts),
       async (args) => {
         await commands.fetch(args.txid!);
       },
     )
 
+    .command(
+      "extract <zip-path>",
+      "Extract a compressed knowledge.",
+      (yargs) => yargs.positional(args.zipPath.id, args.zipPath.opts),
+      async (args) => {
+        await commands.extract(args.zipPath!);
+      },
+    )
+
     .command(
       "set-contract <contract>",
       "Set default contract.",
-      (yargs) => yargs.option(contractIdArg.id, { ...contractIdArg.opts, demandOption: true }),
+      (yargs) => yargs.option(args.contractId.id, { ...args.contractId.opts, demandOption: true }),
       (args) => {
         setConfig({
           contract: args.contract,

src/commands/extract.ts

@@ -0,0 +1,15 @@
+import { extractZip } from "../common";
+import constants from "../constants";
+
+/**
+ * Extracts a zipped knowledge.
+ *
+ * This command is used in particular when you have a large knowledge and you would like to
+ * move it around locally, without downloading the entire thing again.
+ *
+ * @param path Arweave txID to download
+ */
+export default async function cmdExtract(path: string) {
+  const outDir = constants.DRIA.DATA;
+  await extractZip(path, outDir);
+}

src/commands/fetch.ts

@@ -1,4 +1,4 @@
-import { downloadAndUnzip } from "../common";
+import { downloadZip, extractZip } from "../common";
 import constants from "../constants";
 
 /**
@@ -18,5 +18,7 @@ import constants from "../constants";
  * @param txId Arweave txID to download
  */
 export default async function cmdFetch(txId: string) {
-  await downloadAndUnzip(txId, constants.DRIA.DATA);
+  const outDir = constants.DRIA.DATA;
+  const zipPath = await downloadZip(txId);
+  await extractZip(zipPath, outDir);
 }

src/commands/index.ts

@@ -1,5 +1,6 @@
 import pull from "./pull";
 import fetch from "./fetch";
+import extract from "./extract";
 import stop from "./stop";
 import list from "./list";
 import serve from "./serve";
@@ -8,6 +9,7 @@ import clear from "./clear";
 export default {
   pull,
   fetch,
+  extract,
   stop,
   list,
   serve,

src/common/download.ts

@@ -1,17 +1,13 @@
 import Axios from "axios";
-import unzipper from "unzipper";
 import constants from "../constants";
 import { logger } from ".";
-import { createReadStream, createWriteStream, existsSync, mkdirSync, rmSync } from "fs";
+import { createWriteStream, existsSync, mkdirSync } from "fs";
 
-/** Download a zipped data from Arweave, unzip & extract it at a given path.
- *
- * Uses streaming to write request to tmp disk, and then to target folder due to large size.
+/** Download a zipped data from Arweave and writes to dist with streams.
  *
  * @param txId txID on Arweave
- * @param outDir output directory for the extraction file
  */
-export async function downloadAndUnzip(txId: string, outDir: string) {
+export async function downloadZip(txId: string) {
   const url = `${constants.ARWEAVE.DOWNLOAD_URL}/${txId}`;
 
   logger.info("Downloading from", url);
@@ -20,8 +16,8 @@ export async function downloadAndUnzip(txId: string, outDir: string) {
   if (!existsSync(constants.DRIA.TMP)) {
     mkdirSync(constants.DRIA.TMP, { recursive: true });
   }
-  const tmpPath = `${constants.DRIA.TMP}/${txId}.zip`;
-  const writer = createWriteStream(tmpPath);
+  const zipPath = `${constants.DRIA.TMP}/${txId}.zip`;
+  const writer = createWriteStream(zipPath);
   await Axios({
     url,
     method: "get",
@@ -36,7 +32,7 @@ export async function downloadAndUnzip(txId: string, outDir: string) {
       }
     },
   }).then((response) => {
-    return new Promise((resolve, reject) => {
+    return new Promise<void>((resolve, reject) => {
       response.data.pipe(writer);
 
       let error: Error | null = null;
@@ -48,36 +44,11 @@ export async function downloadAndUnzip(txId: string, outDir: string) {
       });
 
       writer.on("close", () => {
-        if (!error) resolve(true);
+        if (!error) resolve();
         // if error, we've rejected above
       });
     });
   });
 
-  try {
-    await new Promise((resolve, reject) => {
-      createReadStream(tmpPath)
-        // unzips to out directory
-        .pipe(unzipper.Extract({ path: outDir, verbose: process.env.NODE_ENV !== "test" }))
-        .on("error", (err) => {
-          reject(err);
-        })
-        .on("close", () => {
-          logger.info("Knowledge extracted at", outDir);
-          logger.info("Cleaning up zip artifacts.");
-          rmSync(tmpPath);
-          logger.info("Done.");
-          resolve(true);
-        });
-    });
-  } catch (err) {
-    logger.error((err as Error).toString());
-
-    logger.info(`Something went wrong while extracting the downloaded zip file.
-You can instead try unzipping via:
-
-  unzip ~/.dria/tmp/${txId}.zip -d ~/.dria/data
-
-`);
-  }
+  return zipPath;
 }

src/common/extract.ts

@@ -0,0 +1,38 @@
+import unzipper from "unzipper";
+import { createReadStream, rmSync } from "fs";
+import { logger } from ".";
+
+/**
+ * Unzips a zip file.
+ *
+ * @param zipPath path to zip file
+ * @param outDir path to extraction
+ */
+export async function extractZip(zipPath: string, outDir: string) {
+  try {
+    await new Promise<void>((resolve, reject) => {
+      createReadStream(zipPath)
+        // unzips to out directory
+        .pipe(unzipper.Extract({ path: outDir, verbose: process.env.NODE_ENV !== "test" }))
+        .on("error", (err) => {
+          reject(err);
+        })
+        .on("close", () => {
+          logger.info("Knowledge extracted at", outDir);
+          logger.info("Cleaning up zip artifacts.");
+          rmSync(zipPath);
+          logger.info("Done.");
+          resolve();
+        });
+    });
+  } catch (err) {
+    logger.error((err as Error).toString());
+
+    logger.info(`Something went wrong while extracting the downloaded zip file.
+You can instead try unzipping via:
+
+  unzip ${zipPath} -d ~/.dria/data
+
+`);
+  }
+}

src/common/index.ts

@@ -4,6 +4,7 @@ import constants from "../constants";
 
 export * from "./download";
 export * from "./image";
+export * from "./extract";
 export * from "./container";
 export * from "./network";