Remove ipfs types, simplify code and get tests passing

packages/common/src/project/IpfsHttpClientLite/IPFSHTTPClientLite.ts

@@ -1,28 +1,40 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import {addAll} from '@subql/common/project/IpfsHttpClientLite/addAll';
 import axios from 'axios';
 import FormData from 'form-data';
-import type {Pin} from 'ipfs-core-types/types/src/pin/remote/index';
-import type {AddOptions, AddResult} from 'ipfs-core-types/types/src/root';
-import type {AbortOptions, ImportCandidate} from 'ipfs-core-types/types/src/utils';
-import type {HTTPClientExtraOptions} from 'ipfs-http-client/types/src/types';
-import {CID} from 'multiformats/cid';
-import {LiteAddAllOptions, IPFSOptions} from './interfaces';
-import {asyncIterableFromStream, decodePin} from './utils';
+import {asyncIterableFromStream} from './utils';
 
-export class IPFSHTTPClientLite {
-  private option: IPFSOptions;
+type ContentData = string | Uint8Array;
+
+type Content =
+  | {
+      content: ContentData;
+      path: string;
+    }
+  | ContentData;
+
+type AddOptions = {
+  pin?: boolean;
+  cidVersion?: number;
+  wrapWithDirectory?: boolean;
+};
 
-  constructor(option: IPFSOptions) {
+type AddResult = {
+  path: string;
+  cid: string;
+  size: number;
+};
+
+export class IPFSHTTPClientLite {
+  constructor(private option: {url: string; headers?: Record<string, string>}) {
+    if (option.url === undefined) {
+      throw new Error('url is required');
+    }
     this.option = option;
   }
 
   get url(): string {
-    if (this.option.url === undefined) {
-      throw new Error('url is required');
-    }
     return this.option.url.toString();
   }
 
@@ -45,43 +57,30 @@ export class IPFSHTTPClientLite {
   /**
    * Import a file or data into IPFS
    */
-  async add(content: ImportCandidate, options?: AddOptions): Promise<AddResult> {
-    const addUrl = `${this.url}/add`;
-    const data = new FormData();
-    if (content instanceof Uint8Array) {
-      content = Buffer.from(content);
-    }
-    data.append('data', content);
-    const response = await axios.post(addUrl, data, {
-      headers: {
-        ...data.getHeaders(),
-        ...this.option.headers,
-      },
-      params: options,
-    });
-
-    const {Hash, Path, Size} = response.data;
-    return {cid: CID.parse(Hash), size: Size, path: Path};
+  async add(content: Content, options?: AddOptions): Promise<AddResult> {
+    const results = await this.addAll([content], options);
+
+    return results[0];
   }
 
   /**
    * Pin a content with a given CID to a remote pinning service.
    */
-  async pinRemoteAdd(cid: CID, options: {service: string}): Promise<Pin> {
-    const addUrl = `${this.url}/pin/remote/add`;
-    // For our own use, we only limited to these args
-    const urlWithParams = `${addUrl}?arg=${cid}&service=${options.service}`;
+  async pinRemoteAdd(cid: string, options: {service: string}): Promise<{Cid: string; Name: string; Status: string}> {
+    const url = new URL(`${this.url}/pin/remote/add`);
+    url.searchParams.append('arg', cid);
+    url.searchParams.append('service', options.service);
     try {
       const response = await axios.post(
-        urlWithParams,
+        url.toString(),
         {},
         {
           headers: {
             ...this.option.headers,
           },
         }
       );
-      return decodePin(response.data);
+      return response.data;
     } catch (e) {
       throw new Error(`Failed to pin CID ${cid} to remote service`, {cause: e});
     }
@@ -90,12 +89,57 @@ export class IPFSHTTPClientLite {
   /**
    * Import multiple files and data into IPFS
    */
+  async addAll(source: Content[], options?: AddOptions): Promise<AddResult[]> {
+    const formData = this.makeFormData(source);
+
+    const url = new URL(`${this.url}/add`);
+    if (options) {
+      url.searchParams.append('pin', options.pin?.toString() ?? 'true');
+      url.searchParams.append('cid-version', options.cidVersion?.toString() ?? '0');
+      url.searchParams.append('wrap-with-directory', options.wrapWithDirectory?.toString() ?? 'false');
+    }
+
+    try {
+      const response = await axios.post(url.toString(), formData, {
+        headers: {
+          ...this.option.headers,
+          ...formData.getHeaders(),
+        },
+        maxBodyLength: Infinity,
+        maxContentLength: Infinity,
+      });
+
+      const mapResponse = (raw: any): AddResult => ({
+        path: raw.Name,
+        cid: raw.Hash,
+        size: parseInt(raw.Size, 10),
+      });
+
+      // If only one file is uploaded then the response is an object.
+      if (typeof response.data === 'object') {
+        return [mapResponse(response.data)];
+      }
+
+      const jsonLines = (response.data.split('\n') as string[]).filter((l) => l !== '');
+
+      return jsonLines.map((line) => JSON.parse(line)).map(mapResponse);
+    } catch (error) {
+      throw new Error(`Failed to upload files to IPFS`, {cause: error});
+    }
+  }
+
+  private makeFormData(contents: Content[]): FormData {
+    const formData = new FormData();
+    for (const content of contents) {
+      if (content instanceof Uint8Array) {
+        formData.append('data', Buffer.from(content));
+      } else if (typeof content === 'string') {
+        formData.append('data', content);
+      } else {
+        formData.append('data', content.content, {filename: content.path});
+      }
+    }
 
-  addAll(
-    source: ImportCandidate[],
-    options?: LiteAddAllOptions & AbortOptions & HTTPClientExtraOptions
-  ): AsyncIterable<AddResult> {
-    const addUrl = `${this.url}/add`;
-    return addAll(addUrl, source, {...options, headers: this.option.headers});
+    return formData;
   }
 }

packages/common/src/project/IpfsHttpClientLite/IpfsHttpClientLite.test.ts → packages/common/src/project/IpfsHttpClientLite/IpfsHttpClientLite.spec.ts

@@ -3,7 +3,6 @@
 
 import {u8aConcat} from '@polkadot/util';
 import {create, IPFSHTTPClient} from 'ipfs-http-client';
-import {CID} from 'multiformats/cid';
 import {IPFS_NODE_ENDPOINT, IPFS_WRITE_ENDPOINT} from '../../constants';
 import {IPFSHTTPClientLite} from './IPFSHTTPClientLite';
 
@@ -12,15 +11,11 @@ const testAuth = process.env.SUBQL_ACCESS_TOKEN!;
 describe('SubIPFSClient', () => {
   let readClient: IPFSHTTPClientLite;
   let writeClient: IPFSHTTPClientLite;
-  let originalReadClient: IPFSHTTPClient;
   let originalWriteClient: IPFSHTTPClient;
 
   beforeAll(() => {
     readClient = new IPFSHTTPClientLite({url: IPFS_NODE_ENDPOINT});
     writeClient = new IPFSHTTPClientLite({url: IPFS_WRITE_ENDPOINT, headers: {Authorization: `Bearer ${testAuth}`}});
-    originalReadClient = create({
-      url: IPFS_NODE_ENDPOINT,
-    });
     originalWriteClient = create({
       url: IPFS_WRITE_ENDPOINT,
       headers: {Authorization: `Bearer ${testAuth}`},
@@ -38,9 +33,9 @@ describe('SubIPFSClient', () => {
     for await (const result of originalResults) {
       originalOutput.set(result.path, result.cid.toString());
     }
-    const results = writeClient.addAll(source, {pin: true, cidVersion: 0, wrapWithDirectory: false});
+    const results = await writeClient.addAll(source, {pin: true, cidVersion: 0, wrapWithDirectory: false});
     const output: Map<string, string> = new Map();
-    for await (const result of results) {
+    for (const result of results) {
       output.set(result.path, result.cid.toString());
     }
     expect(originalOutput).toEqual(output);
@@ -72,8 +67,8 @@ describe('SubIPFSClient', () => {
   //
   it('should pin a content with given CID to a remote pinning service', async () => {
     const testCID = 'QmQKeYj2UZJoTN5yXSvzJy4A3CjUuSmEWAKeZV4herh5bS';
-    const result = await writeClient.pinRemoteAdd(CID.parse(testCID), {service: 'onfinality'});
-    expect(result.cid.toString()).toBe(testCID);
+    const result = await writeClient.pinRemoteAdd(testCID, {service: 'onfinality'});
+    expect(result.Cid).toBe(testCID);
   });
 });
 

packages/common/src/project/IpfsHttpClientLite/utils.ts

@@ -2,20 +2,10 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {AxiosResponse} from 'axios';
-import type {Pin, Status} from 'ipfs-core-types/types/src/pin/remote/index';
-import {CID} from 'multiformats/cid';
 
 export async function* asyncIterableFromStream(response: Promise<AxiosResponse>): AsyncIterable<Uint8Array> {
   const stream = (await response).data;
   for await (const chunk of stream) {
     yield new Uint8Array(Buffer.from(chunk));
   }
 }
-
-export const decodePin = ({Cid: cid, Name: name, Status: status}: {Cid: string; Name: string; Status: Status}): Pin => {
-  return {
-    cid: CID.parse(cid),
-    name,
-    status,
-  };
-};