Feat: Idempotent CSV Serialization

example-app/src/admin/resources/user/user.resource.ts

@@ -17,6 +17,21 @@ export const createUserResource = (
   features: [
     importExportFeature({
       componentLoader,
+      properties: {
+        import: {
+          csv: {
+            nullValue: 'null',
+            undefinedValue: 'undefined',
+          },
+          upsertById: true,
+        },
+        export: {
+          csv: {
+            nullValue: 'null',
+            undefinedValue: 'undefined',
+          },
+        },
+      },
     }),
   ],
 });

src/export.handler.ts

@@ -2,18 +2,18 @@ import { ActionHandler, ActionResponse } from 'adminjs';
 
 import { Parsers } from './parsers.js';
 import { getRecords } from './utils.js';
+import { ImportExportFeatureOptions } from './importExportFeature.js';
 
-export const exportHandler: ActionHandler<ActionResponse> = async (
-  request,
-  response,
-  context
-) => {
-  const parser = Parsers[request.query?.type ?? 'json'].export;
+export const exportHandler: (
+  options: ImportExportFeatureOptions
+) => ActionHandler<ActionResponse> =
+  options => async (request, response, context) => {
+    const parser = Parsers[request.query?.type ?? 'json'].export;
 
-  const records = await getRecords(context);
-  const parsedData = parser(records);
+    const records = await getRecords(context);
+    const parsedData = parser(records, options);
 
-  return {
-    exportedData: parsedData,
-  };
+    return {
+      exportedData: parsedData,
+    };
 };

src/import.handler.ts

@@ -3,19 +3,19 @@ import fs from 'fs';
 import util from 'util';
 
 import { getFileFromRequest, getImporterByFileName } from './utils.js';
+import { ImportExportFeatureOptions } from './importExportFeature.js';
 
 const readFile = util.promisify(fs.readFile);
 
-export const importHandler: ActionHandler<ActionResponse> = async (
-  request,
-  response,
-  context
-) => {
-  const file = getFileFromRequest(request);
-  const importer = getImporterByFileName(file.name);
+export const importHandler: (
+  options: ImportExportFeatureOptions
+) => ActionHandler<ActionResponse> =
+  options => async (request, response, context) => {
+    const file = getFileFromRequest(request);
+    const importer = getImporterByFileName(file.name);
 
-  const fileContent = await readFile(file.path);
-  await importer(fileContent.toString(), context.resource);
+    const fileContent = await readFile(file.path);
+    await importer(fileContent.toString(), context.resource, options);
 
-  return {};
-};
+    return {};
+  };

src/importExportFeature.ts

@@ -5,11 +5,54 @@ import { exportHandler } from './export.handler.js';
 import { importHandler } from './import.handler.js';
 import { bundleComponent } from './bundle-component.js';
 
-type ImportExportFeatureOptions = {
+export type ImportExportFeatureOptions = {
   /**
-   * Your ComponentLoader instance. It is required for the feature to add it's components.
+   * Your ComponentLoader instance. It is required for the feature to add its components.
    */
   componentLoader: ComponentLoader;
+
+  /**
+   * Names of the properties used by the feature
+   */
+  properties?: {
+    /**
+     * Optional export configuration
+     */
+    export?: {
+      /**
+       * CSV export configuration
+       */
+      csv?: {
+        /**
+         * In CSV export, convert `null` to this (default: '')
+         */
+        nullValue?: string;
+        /**
+         * In CSV export, convert `undefined` to this (default: '')
+         */
+        undefinedValue?: string;
+      };
+    };
+
+    import?: {
+      csv: {
+        /**
+         * In CSV import, convert this string to `undefined`
+         */
+        undefinedValue?: string;
+
+        /**
+         * In CSV import, convert this string to `null`
+         */
+        nullValue?: string;
+      };
+
+      /**
+       * During import, upsert records by ID rather than create
+       */
+      upsertById: boolean;
+    };
+  };
 };
 
 const importExportFeature = (
@@ -22,12 +65,12 @@ const importExportFeature = (
   return buildFeature({
     actions: {
       export: {
-        handler: postActionHandler(exportHandler),
+        handler: postActionHandler(exportHandler(options)),
         component: exportComponent,
         actionType: 'resource',
       },
       import: {
-        handler: postActionHandler(importHandler),
+        handler: postActionHandler(importHandler(options)),
         component: importComponent,
         actionType: 'resource',
       },

src/modules/csv/csv.exporter.ts

@@ -1,6 +1,15 @@
-import { BaseRecord } from 'adminjs';
 import { parse } from 'json2csv';
+import { Exporter } from '../../parsers.js';
+import { emptyValuesTransformer } from '../transformers/empty-values.transformer.js';
 
-export const csvExporter = (records: BaseRecord[]): string => {
-  return parse(records.map(r => r.params));
+export const csvExporter: Exporter = (records, options) => {
+  return parse(
+    records.map(record =>
+      emptyValuesTransformer(
+        record.params,
+        'export',
+        options?.properties?.export?.csv
+      )
+    )
+  );
 };

src/modules/csv/csv.importer.ts

@@ -2,9 +2,16 @@ import csv from 'csvtojson';
 
 import { Importer } from '../../parsers.js';
 import { saveRecords } from '../../utils.js';
+import { emptyValuesTransformer } from '../transformers/empty-values.transformer.js';
+
+export const csvImporter: Importer = async (csvString, resource, options) => {
+  const importProperties = options?.properties?.import?.csv;
 
-export const csvImporter: Importer = async (csvString, resource) => {
   const records = await csv().fromString(csvString);
 
-  return saveRecords(records, resource);
+  const transformedRecords = records.map(record =>
+    emptyValuesTransformer(record, 'import', importProperties)
+  );
+
+  return saveRecords(transformedRecords, resource, options);
 };

src/modules/json/json.exporter.ts

@@ -1,5 +1,5 @@
-import { BaseRecord } from 'adminjs';
+import { Exporter } from '../../parsers.js';
 
-export const jsonExporter = (records: BaseRecord[]): string => {
+export const jsonExporter: Exporter = (records, options) => {
   return JSON.stringify(records.map(r => r.params));
 };

src/modules/json/json.importer.ts

@@ -1,8 +1,13 @@
 import { Importer } from '../../parsers.js';
 import { saveRecords } from '../../utils.js';
+import { ImportExportFeatureOptions } from '../../importExportFeature.js';
 
-export const jsonImporter: Importer = async (jsonString, resource) => {
+export const jsonImporter: Importer = async (
+  jsonString,
+  resource,
+  options: ImportExportFeatureOptions
+) => {
   const records = JSON.parse(jsonString);
 
-  return saveRecords(records, resource);
+  return saveRecords(records, resource, options);
 };

src/modules/transformers/empty-values.transformer.ts

@@ -0,0 +1,33 @@
+export const emptyValuesTransformer: (
+  record: Record<string, any>,
+  operation: 'import' | 'export',
+  options?: { undefinedValue?: string; nullValue?: string }
+) => Record<string, any> = (record, operation, options = {}) => {
+  if (!options?.nullValue && !options?.undefinedValue) {
+    return record;
+  }
+
+  const { nullValue, undefinedValue } = options;
+  const transformedEntries = Object.entries(record).map(([key, value]) => {
+    if (operation === 'export') {
+      if (nullValue !== undefined && value === null) {
+        return [key, nullValue];
+      }
+      if (undefinedValue !== undefined && value === undefined) {
+        return [key, undefinedValue];
+      }
+    }
+    if (operation === 'import') {
+      if (nullValue !== undefined && value === nullValue) {
+        return [key, null];
+      }
+      if (undefinedValue !== undefined && value === undefinedValue) {
+        return [key, undefined];
+      }
+    }
+
+    return [key, value];
+  });
+
+  return Object.fromEntries(transformedEntries);
+};

src/modules/xml/xml.exporter.ts

@@ -1,7 +1,7 @@
-import { BaseRecord } from 'adminjs';
 import xml from 'xml';
+import { Exporter } from '../../parsers.js';
 
-export const xmlExporter = (records: BaseRecord[]): string => {
+export const xmlExporter: Exporter = (records, options) => {
   const data = records.map(record => ({
     record: Object.entries(record.params).map(([key, value]) => ({
       [key]: value,

src/modules/xml/xml.importer.ts

@@ -3,11 +3,11 @@ import xml2js from 'xml2js';
 import { Importer } from '../../parsers.js';
 import { saveRecords } from '../../utils.js';
 
-export const xmlImporter: Importer = async (xmlString, resource) => {
+export const xmlImporter: Importer = async (xmlString, resource, options) => {
   const parser = new xml2js.Parser({ explicitArray: false });
   const {
     records: { record },
   } = await parser.parseStringPromise(xmlString);
 
-  return saveRecords(record, resource);
+  return saveRecords(record, resource, options);
 };

src/parsers.ts

@@ -7,12 +7,17 @@ import { csvExporter } from './modules/csv/csv.exporter.js';
 import { xmlExporter } from './modules/xml/xml.exporter.js';
 import { csvImporter } from './modules/csv/csv.importer.js';
 import { xmlImporter } from './modules/xml/xml.importer.js';
+import { ImportExportFeatureOptions } from './importExportFeature.js';
 
-export type Exporter = (records: BaseRecord[]) => string;
+export type Exporter = (
+  records: BaseRecord[],
+  options: ImportExportFeatureOptions
+) => string;
 
 export type Importer = (
   records: string,
-  resource: BaseResource
+  resource: BaseResource,
+  options: ImportExportFeatureOptions
 ) => Promise<BaseRecord[]>;
 
 export const Parsers: Record<

src/utils.ts

@@ -13,8 +13,20 @@ import { csvImporter } from './modules/csv/csv.importer.js';
 import { jsonImporter } from './modules/json/json.importer.js';
 import { xmlImporter } from './modules/xml/xml.importer.js';
 import { Importer } from './parsers.js';
+import { ImportExportFeatureOptions } from './importExportFeature.js';
 
 export const saveRecords = async (
+  records: Record<string, any>[],
+  resource: BaseResource,
+  options?: ImportExportFeatureOptions
+): Promise<BaseRecord[]> => {
+  if (!options?.properties?.import?.upsertById) {
+    return createRecords(records, resource);
+  }
+  return upsertRecords(records, resource);
+};
+
+const createRecords = async (
   records: Record<string, any>[],
   resource: BaseResource
 ): Promise<BaseRecord[]> => {
@@ -30,6 +42,39 @@ export const saveRecords = async (
   );
 };
 
+const upsertRecords = async (
+  records: Record<string, any>[],
+  resource: BaseResource
+): Promise<BaseRecord[]> => {
+  debugger;
+  const idFieldName =
+    resource
+      .properties()
+      .find(property => property.isId())
+      ?.name?.() || 'id';
+  const ids = records.map(records => records[idFieldName]).filter(Boolean);
+  const existingRecords = await resource.findMany(ids);
+  const knownIds = new Set(
+    existingRecords.map(record => record.params[idFieldName])
+  );
+  return Promise.all(
+    records.map(async record => {
+      debugger;
+      try {
+        const recordId = record[idFieldName];
+        if (knownIds.has(recordId)) {
+          return resource.update(recordId, record);
+        }
+
+        return resource.create(record);
+      } catch (e) {
+        console.error(e);
+        return e;
+      }
+    })
+  );
+};
+
 export const getImporterByFileName = (fileName: string): Importer => {
   if (fileName.includes('.json')) {
     return jsonImporter;