Merge pull request #20 from dolthub/taylor/file-upload

Add file upload

packages/graphql-server/package.json

@@ -37,6 +37,7 @@
     "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
     "graphql": "^16.7.1",
+    "graphql-upload": "13",
     "mysql2": "^3.1.0",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.4",

packages/graphql-server/schema.gql

@@ -201,6 +201,24 @@ type Mutation {
   deleteBranch(databaseName: String!, branchName: String!): Boolean!
   addDatabaseConnection(url: String, useEnv: Boolean): String!
   createDatabase(databaseName: String!): Boolean!
+  loadDataFile(tableName: String!, refName: String!, databaseName: String!, importOp: ImportOperation!, fileType: FileType!, file: Upload!, modifier: LoadDataModifier): Boolean!
   createTag(tagName: String!, databaseName: String!, message: String, fromRefName: String!): Tag!
   deleteTag(databaseName: String!, tagName: String!): Boolean!
+}
+
+enum ImportOperation {
+  Update
+}
+
+enum FileType {
+  Csv
+  Psv
+}
+
+"""The `Upload` scalar type represents a file upload."""
+scalar Upload
+
+enum LoadDataModifier {
+  Ignore
+  Replace
 }

packages/graphql-server/src/dataSources/dataSource.module.ts

@@ -5,7 +5,7 @@ import { DataSourceService } from "./dataSource.service";
   providers: [
     {
       provide: DataSourceService,
-      useValue: new DataSourceService(undefined),
+      useValue: new DataSourceService(undefined, undefined),
     },
   ],
   exports: [DataSourceService],

packages/graphql-server/src/dataSources/dataSource.service.ts

@@ -1,4 +1,5 @@
 import { Injectable } from "@nestjs/common";
+import * as mysql from "mysql2/promise";
 import { DataSource, QueryRunner } from "typeorm";
 import { RawRows } from "../utils/commonTypes";
 
@@ -7,14 +8,23 @@ export type ParQuery = (q: string, p?: any[] | undefined) => Promise<RawRows>;
 
 @Injectable()
 export class DataSourceService {
-  constructor(private ds: DataSource | undefined) {}
+  constructor(
+    private ds: DataSource | undefined,
+    private mysqlConfig: mysql.ConnectionOptions | undefined, // Used for file upload
+  ) {}
 
   getDS(): DataSource {
     const { ds } = this;
     if (!ds) throw new Error(dbNotFoundErr);
     return ds;
   }
 
+  getMySQLConfig(): mysql.ConnectionOptions {
+    const { mysqlConfig } = this;
+    if (!mysqlConfig) throw new Error("MySQL config not found");
+    return mysqlConfig;
+  }
+
   getQR(): QueryRunner {
     return this.getDS().createQueryRunner();
   }
@@ -94,13 +104,25 @@ export class DataSourceService {
       },
     });
 
+    this.mysqlConfig = {
+      uri: connUrl,
+      ssl: {
+        rejectUnauthorized: false,
+      },
+      connectionLimit: 1,
+      dateStrings: ["DATE"],
+
+      // Allows file upload via LOAD DATA
+      flags: ["+LOCAL_FILES"],
+    };
+
     await this.ds.initialize();
   }
 }
 
 // Cannot use params here for the database revision. It will incorrectly
 // escape refs with dots
-function useDBStatement(
+export function useDBStatement(
   dbName?: string,
   refName?: string,
   isDolt = true,

packages/graphql-server/src/main.ts

@@ -1,11 +1,16 @@
 import { NestFactory } from "@nestjs/core";
 import * as cookieParser from "cookie-parser";
 import * as cors from "cors";
+import { graphqlUploadExpress } from "graphql-upload";
 import { AppModule } from "./app.module";
 
+const oneMB = 1024 * 1024;
+const maxFileSize = 400 * oneMB;
+
 async function bootstrap() {
   const app = await NestFactory.create(AppModule);
   app.use(cookieParser());
+  app.use(graphqlUploadExpress({ maxFileSize, maxFiles: 1 }));
   if (process.env.NODE_ENV === "development") {
     app.use(
       "/graphql",

packages/graphql-server/src/resolvers.ts

@@ -6,13 +6,15 @@ import { RowResolver } from "./rows/row.resolver";
 import { SqlSelectResolver } from "./sqlSelects/sqlSelect.resolver";
 import { StatusResolver } from "./status/status.resolver";
 import { TableResolver } from "./tables/table.resolver";
+import { FileUploadResolver } from "./tables/upload.resolver";
 import { TagResolver } from "./tags/tag.resolver";
 
 const resolvers = [
   BranchResolver,
   CommitResolver,
   DatabaseResolver,
   DocsResolver,
+  FileUploadResolver,
   RowResolver,
   SqlSelectResolver,
   StatusResolver,

packages/graphql-server/src/tables/table.enum.ts

@@ -0,0 +1,28 @@
+import { registerEnumType } from "@nestjs/graphql";
+
+export enum ImportOperation {
+  // Create,
+  // Overwrite,
+  Update,
+  // Replace,
+}
+
+registerEnumType(ImportOperation, { name: "ImportOperation" });
+
+export enum FileType {
+  Csv,
+  Psv,
+  // Xlsx,
+  // Json,
+  // Sql,
+  // Any,
+}
+
+registerEnumType(FileType, { name: "FileType" });
+
+export enum LoadDataModifier {
+  Ignore,
+  Replace,
+}
+
+registerEnumType(LoadDataModifier, { name: "LoadDataModifier" });

packages/graphql-server/src/tables/table.queries.ts

@@ -1,3 +1,5 @@
+import { FileType, LoadDataModifier } from "./table.enum";
+
 export const indexQuery = `SELECT 
   table_name, 
   index_name, 
@@ -13,3 +15,32 @@ export const foreignKeysQuery = `SELECT * FROM INFORMATION_SCHEMA.KEY_COLUMN_USA
 export const columnsQuery = `DESCRIBE ??`;
 
 export const listTablesQuery = `SHOW FULL TABLES WHERE table_type = 'BASE TABLE'`;
+
+export const getLoadDataQuery = (
+  filename: string,
+  tableName: string,
+  fileType: FileType,
+  modifier?: LoadDataModifier,
+): string => `LOAD DATA LOCAL INFILE '${filename}'
+${getModifier(modifier)}INTO TABLE \`${tableName}\` 
+FIELDS TERMINATED BY '${getDelim(fileType)}' ENCLOSED BY '' 
+LINES TERMINATED BY '\n' 
+IGNORE 1 ROWS;`;
+
+function getModifier(m?: LoadDataModifier): string {
+  switch (m) {
+    case LoadDataModifier.Ignore:
+      return "IGNORE ";
+    case LoadDataModifier.Replace:
+      return "REPLACE ";
+    default:
+      return "";
+  }
+}
+
+function getDelim(ft: FileType): string {
+  if (ft === FileType.Psv) {
+    return "|";
+  }
+  return ",";
+}

packages/graphql-server/src/tables/upload.resolver.ts

@@ -0,0 +1,71 @@
+import { Args, ArgsType, Field, Mutation, Resolver } from "@nestjs/graphql";
+import { ReadStream } from "fs";
+import { GraphQLUpload } from "graphql-upload";
+import * as mysql from "mysql2/promise";
+import {
+  DataSourceService,
+  useDBStatement,
+} from "../dataSources/dataSource.service";
+import { TableArgs } from "../utils/commonTypes";
+import { FileType, ImportOperation, LoadDataModifier } from "./table.enum";
+import { Table } from "./table.model";
+import { getLoadDataQuery } from "./table.queries";
+
+export interface FileUpload {
+  filename: string;
+  mimetype: string;
+  encoding: string;
+  createReadStream: () => ReadStream;
+}
+
+@ArgsType()
+class TableImportArgs extends TableArgs {
+  @Field(_type => ImportOperation)
+  importOp: ImportOperation;
+
+  @Field(_type => FileType)
+  fileType: FileType;
+
+  @Field(() => GraphQLUpload)
+  file: Promise<FileUpload>;
+
+  @Field(_type => LoadDataModifier, { nullable: true })
+  modifier?: LoadDataModifier;
+}
+
+@Resolver(_of => Table)
+export class FileUploadResolver {
+  constructor(private readonly dss: DataSourceService) {}
+
+  @Mutation(_returns => Boolean)
+  async loadDataFile(@Args() args: TableImportArgs): Promise<boolean> {
+    const conn = await mysql.createConnection(this.dss.getMySQLConfig());
+
+    let isDolt = false;
+    try {
+      const res = await conn.query("SELECT dolt_version()");
+      isDolt = !!res;
+    } catch (_) {
+      // ignore
+    }
+
+    await conn.query(useDBStatement(args.databaseName, args.refName, isDolt));
+    await conn.query("SET GLOBAL local_infile=ON;");
+
+    const { createReadStream, filename } = await args.file;
+
+    await conn.query({
+      sql: getLoadDataQuery(
+        filename,
+        args.tableName,
+        args.fileType,
+        args.modifier,
+      ),
+      infileStreamFactory: createReadStream,
+    });
+
+    conn.destroy();
+
+    return true;
+  }
+}

packages/web/components/CustomFormSelect/DoltDisabledSelector.tsx

@@ -12,12 +12,12 @@ export default function DoltDisabledSelector(props: Props) {
       <div
         className={css.doltDisabled}
         data-tooltip-content="Use Dolt to enable branches"
-        data-tooltip-id="branch-selector-no-dolt"
+        data-tooltip-id="selector-no-dolt"
         data-tooltip-place="top"
       >
         {props.val}
       </div>
-      <Tooltip id="branch-selector-no-dolt" />
+      <Tooltip id="selector-no-dolt" />
     </>
   );
 }

packages/web/components/CustomRadio/index.module.css

@@ -0,0 +1,44 @@
+.container {
+  @apply block relative pl-8 mb-2 text-primary font-semibold cursor-pointer select-none;
+
+  input {
+    @apply absolute opacity-0;
+  }
+}
+
+.disabled {
+  @apply text-ld-darkgrey cursor-default;
+}
+
+.checkmark {
+  @apply absolute top-0 left-0 bg-white rounded-full mt-1 border border-primary w-4 h-4;
+}
+
+.container:hover input ~ .checkmark {
+  @apply border-acc-linkblue;
+}
+
+.container input:checked ~ .checkmark {
+  @apply bg-white;
+}
+.container input:focus ~ .checkmark {
+  @apply widget-shadow-lightblue;
+}
+
+.container input:disabled ~ .checkmark,
+.container:hover input:disabled ~ .checkmark {
+  @apply border-ld-darkgrey;
+}
+
+.checkmark:after {
+  @apply absolute hidden;
+  content: "";
+}
+
+.container .checkmark:after {
+  @apply rounded-full bg-white top-[3px] left-[3px] w-2 h-2;
+}
+
+.container input:checked ~ .checkmark:after {
+  @apply block bg-primary;
+}

packages/web/components/CustomRadio/index.test.tsx

@@ -0,0 +1,50 @@
+import { setup } from "@lib/testUtils.test";
+import { screen } from "@testing-library/react";
+import CustomRadio from ".";
+
+describe("test CustomCheckbox", () => {
+  const mocks = [
+    { name: "one", label: "one-label" },
+    { name: "two", label: "two-label" },
+    { name: "three", label: "three-label" },
+  ];
+
+  mocks.forEach((mock, ind) => {
+    it(`renders CustomRadio for of label ${mock.label}`, async () => {
+      const checked = ind % 2 === 0;
+      const disabled = ind === 2;
+      const onChangeValue = jest.fn();
+
+      const { user } = setup(
+        <CustomRadio
+          {...mock}
+          onChange={onChangeValue}
+          checked={checked}
+          className="classname"
+          disabled={disabled}
+        >
+          {mock.label}
+        </CustomRadio>,
+      );
+      const content = screen.getByLabelText(mock.label);
+      expect(content).toBeVisible();
+      if (!disabled) {
+        const input = screen.getByRole("radio");
+        if (checked) {
+          expect(input).toBeChecked();
+        } else {
+          expect(input).not.toBeChecked();
+        }
+
+        await user.click(screen.getByLabelText(mock.label));
+        if (checked) {
+          expect(onChangeValue).not.toHaveBeenCalled();
+        } else {
+          expect(onChangeValue).toHaveBeenCalled();
+        }
+      } else {
+        expect(content).toBeDisabled();
+      }
+    });
+  });
+});