From f0d5192e769a392df84a4e85e245496ac7c0ef31 Mon Sep 17 00:00:00 2001 From: wyh888 <18819467659@163.com> Date: Tue, 14 May 2024 16:47:59 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8D=E5=A4=8D=20close=20#2718=20(#2719)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复下载数据重复 close #2718 * fix: 修复透视表导出报错问题 * test: 提交单测 * refactor: 修改变量名 * refactor: 抽取公用变量、方法到 BaseDataCellCopy 基类 --------- Co-authored-by: Harvey Wang --- .../unit/utils/export/export-pivot-spec.ts | 37 +++++++++++++++++++ .../unit/utils/export/export-table-spec.ts | 35 +++++++++++++++++- packages/s2-core/__tests__/util/index.ts | 28 ++++++++++++++ .../utils/export/copy/base-data-cell-copy.ts | 13 ++++++- .../utils/export/copy/pivot-data-cell-copy.ts | 24 +++++++----- .../src/utils/export/copy/table-copy.ts | 25 ++++++++----- 6 files changed, 140 insertions(+), 22 deletions(-) diff --git a/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts b/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts index 0aa07f6fb2..a07a56309c 100644 --- a/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts +++ b/packages/s2-core/__tests__/unit/utils/export/export-pivot-spec.ts @@ -15,6 +15,7 @@ import { TAB_SEPARATOR, } from '../../../../src/common/constant'; import { asyncGetAllPlainData, PivotSheet } from '../../../../src'; +import { generateRawData } from '../../../util'; import { CopyMIMEType } from '@/common/interface/export'; describe('PivotSheet Export Test', () => { @@ -464,4 +465,40 @@ describe('PivotSheet Export Test', () => { expect(data.split(LINE_SEPARATOR)).toMatchSnapshot(); }, ); + + // https://github.com/antvis/S2/issues/2718 + it('should export the correct amount of data and have no duplicate data', async () => { + const typeCount = 100; + const subTypeCount = 100; + + const bigData = generateRawData( + { province: 10, city: 1 }, + { type: typeCount, subType: subTypeCount }, + ); + + const sheet = new PivotSheet( + getContainer(), + assembleDataCfg({ + data: bigData, + fields: { + rows: ['type', 'subType'], + columns: ['province', 'city'], + values: ['number'], + }, + }), + assembleOptions({}), + ); + + await sheet.render(); + const data = await asyncGetAllPlainData({ + sheetInstance: sheet, + split: CSV_SEPARATOR, + formatOptions: true, + }); + + // row header count + data count + const count = typeCount * subTypeCount + 3; + + expect(data.split(LINE_SEPARATOR)).toHaveLength(count); + }); }); diff --git a/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts b/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts index f1b29deb15..a1eaf3ffd4 100644 --- a/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts +++ b/packages/s2-core/__tests__/unit/utils/export/export-table-spec.ts @@ -1,7 +1,11 @@ /* eslint-disable jest/expect-expect */ import { slice } from 'lodash'; import { data as originData } from 'tests/data/mock-dataset.json'; -import { assembleDataCfg, assembleOptions } from '../../../util'; +import { + assembleDataCfg, + assembleOptions, + generateRawData, +} from '../../../util'; import { createTableSheet, expectMatchSnapshot, @@ -433,4 +437,33 @@ describe('TableSheet Export Test', () => { formatData: false, }); }); + + // https://github.com/antvis/S2/issues/2718 + it('should export the correct amount of data and have no duplicate data', async () => { + const bigData = generateRawData( + { province: 10, city: 10 }, + { type: 10, sub_type: 10 }, + ); + + const tableSheet = new TableSheet( + getContainer(), + assembleDataCfg({ + data: bigData, + fields: { + columns: ['province', 'city', 'type', 'sub_type', 'number'], + }, + }), + assembleOptions(), + ); + + await tableSheet.render(); + const data = await asyncGetAllPlainData({ + sheetInstance: tableSheet, + split: CSV_SEPARATOR, + formatOptions: true, + }); + + // The first line is the header, so the number of lines should be the same as the number of data plus one + expect(data.split(LINE_SEPARATOR)).toHaveLength(bigData.length + 1); + }); }); diff --git a/packages/s2-core/__tests__/util/index.ts b/packages/s2-core/__tests__/util/index.ts index 195f07e5d6..590920438e 100644 --- a/packages/s2-core/__tests__/util/index.ts +++ b/packages/s2-core/__tests__/util/index.ts @@ -34,6 +34,34 @@ export const assembleDataCfg = (...dataCfg: Partial[]) => { return customMerge(DEFAULT_DATA_CONFIG, s2DataCfg, ...dataCfg); }; +export const generateRawData = ( + row: Record, + col: Record, +) => { + const res: Record[] = []; + + const rowKeys = Object.keys(row); + const colKeys = Object.keys(col); + + for (let i = 0; i < row[rowKeys[0]]; i++) { + for (let j = 0; j < row[rowKeys[1]]; j++) { + for (let m = 0; m < col[colKeys[0]]; m++) { + for (let n = 0; n < col[colKeys[1]]; n++) { + res.push({ + province: `p${i}`, + city: `c${j}`, + type: `type${m}`, + subType: `subType${n}`, + number: i * n, + }); + } + } + } + } + + return res; +}; + export const TOTALS_OPTIONS: S2Options['totals'] = { row: { showGrandTotals: true, diff --git a/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts b/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts index 1f26f4db69..e108cf9a88 100644 --- a/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts +++ b/packages/s2-core/src/utils/export/copy/base-data-cell-copy.ts @@ -1,4 +1,8 @@ -import { TAB_SEPARATOR, type DataItem } from '../../../common'; +import { + AsyncRenderThreshold, + TAB_SEPARATOR, + type DataItem, +} from '../../../common'; import type { CopyableHTML, CopyablePlain, @@ -16,6 +20,13 @@ export abstract class BaseDataCellCopy { protected config: CopyAndExportUnifyConfig; + protected idleCallbackCount: number; + + protected initIdleCallbackCount(rowLength: number) { + this.idleCallbackCount = + rowLength >= AsyncRenderThreshold ? AsyncRenderThreshold : rowLength; + } + constructor(params: SheetCopyConstructorParams) { const { spreadsheet, isExport = false, config } = params; diff --git a/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts b/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts index db69af8c6c..3bf0e4ab01 100644 --- a/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts +++ b/packages/s2-core/src/utils/export/copy/pivot-data-cell-copy.ts @@ -8,7 +8,6 @@ import { zip, } from 'lodash'; import { - AsyncRenderThreshold, CornerNodeType, EXTRA_FIELD, VALUE_FIELD, @@ -153,18 +152,20 @@ export class PivotDataCellCopy extends BaseDataCellCopy { // 因为每次 requestIdleCallback 执行的时间不一样，所以需要记录下当前执行到的 this.leafRowNodes 和 this.leafColNodes const dataMatrixIdleCallback = (deadline: IdleDeadline) => { const rowLength: number = this.leafRowNodes.length; + // requestIdleCallback 浏览器空闲时会多次执行, 只有一行数据时执行一次即可, 避免生成重复数据 - let count = - rowLength >= AsyncRenderThreshold - ? AsyncRenderThreshold - : rowLength; + this.initIdleCallbackCount(rowLength); while ( (deadline.timeRemaining() > 0 || deadline.didTimeout) && rowIndex <= rowLength - 1 && - count > 0 + this.idleCallbackCount > 0 ) { - for (let j = rowIndex; j < rowLength && count > 0; j++) { + for ( + let j = rowIndex; + j < rowLength && this.idleCallbackCount > 0; + j++ + ) { const row: DataItem[] = []; const rowNode = this.leafRowNodes[j]; @@ -183,15 +184,18 @@ export class PivotDataCellCopy extends BaseDataCellCopy { row.push(dataItem); } - rowIndex = j; + rowIndex++; matrix.push(row); - count--; + this.idleCallbackCount--; } } - if (rowIndex === rowLength - 1) { + if (rowIndex === rowLength) { resolve(matrix); } else { + // 重置 idleCallbackCount，避免下次 requestIdleCallback 时 idleCallbackCount 为 0 + this.initIdleCallbackCount(rowLength); + requestIdleCallback(dataMatrixIdleCallback); } }; diff --git a/packages/s2-core/src/utils/export/copy/table-copy.ts b/packages/s2-core/src/utils/export/copy/table-copy.ts index 60f045f527..807402755d 100644 --- a/packages/s2-core/src/utils/export/copy/table-copy.ts +++ b/packages/s2-core/src/utils/export/copy/table-copy.ts @@ -1,6 +1,5 @@ import { map, zip } from 'lodash'; import { - AsyncRenderThreshold, SERIES_NUMBER_FIELD, getDefaultSeriesNumberText, type CellMeta, @@ -94,20 +93,22 @@ class TableDataCellCopy extends BaseDataCellCopy { try { const dataMatrixIdleCallback = (deadline: IdleDeadline) => { const rowLength = this.displayData.length; + // requestIdleCallback 浏览器空闲时会多次执行, 只有一行数据时执行一次即可, 避免生成重复数据 - let count = - rowLength >= AsyncRenderThreshold - ? AsyncRenderThreshold - : rowLength; + this.initIdleCallbackCount(rowLength); while ( (deadline.timeRemaining() > 0 || deadline.didTimeout || process.env['NODE_ENV'] === 'test') && rowIndex <= rowLength - 1 && - count > 0 + this.idleCallbackCount > 0 ) { - for (let j = rowIndex; j < rowLength && count > 0; j++) { + for ( + let j = rowIndex; + j < rowLength && this.idleCallbackCount > 0; + j++ + ) { const rowData = this.displayData[j]; const row: string[] = []; @@ -131,15 +132,19 @@ class TableDataCellCopy extends BaseDataCellCopy { row.push(dataItem as string); } - rowIndex = j; + // 生成一行数据后，rowIndex + 1，下次 requestIdleCallback 时从下一行开始 + rowIndex++; result.push(row); - count--; + this.idleCallbackCount--; } } - if (rowIndex === rowLength - 1) { + if (rowIndex === rowLength) { resolve(result); } else { + // 重置 idleCallbackCount，避免下次 requestIdleCallback 时 idleCallbackCount 为 0 + this.initIdleCallbackCount(rowLength); + requestIdleCallback(dataMatrixIdleCallback); } };