diff --git a/client/src/consts/Insert.ts b/client/src/consts/Insert.ts index f1baa8ad..088a88b4 100644 --- a/client/src/consts/Insert.ts +++ b/client/src/consts/Insert.ts @@ -3,4 +3,4 @@ export const INSERT_CSV_SAMPLE = `Date, Country, Units, Revenue,\n 909,3898,[3898...], [84981...]\n ...`; -export const INSERT_MAX_SIZE = 150; +export const INSERT_MAX_SIZE = 256; diff --git a/client/src/i18n/cn/insert.ts b/client/src/i18n/cn/insert.ts index 8c09ff70..e0d1ff5b 100644 --- a/client/src/i18n/cn/insert.ts +++ b/client/src/i18n/cn/insert.ts @@ -7,9 +7,10 @@ const insertTrans = { sample: '样本', noteTitle: '注意', notes: [ - `确保数据中的列名与 Schema 中的字段标签名相同。`, - `数据大小应小于 150MB，行数应小于 100000，以便正确导入数据。`, - `"导入数据" 选项只会添加新记录。您不能使用此选项更新现有记录。`, + `支持 CSV 或者 JSON 文件。`, + `确保 CSV 列名或 JSON key值与 Schema 字段名一致`, + `文件大小不能超过 256MB。`, + `"导入数据" 只会插入新数据，不支持 upsert。`, ], overSizeWarning: '文件数据大小应小于 {{size}}MB', isContainFieldNames: '第一行包含字段名？', @@ -22,8 +23,8 @@ const insertTrans = { previewTipAction: '*更改标题单元格选择器值以编辑字段名', requiredFieldName: '字段名*', - statusLoading: '您的数据正在导入中...可能需要几分钟', - statusLoadingTip: '请耐心等待，谢谢', + statusLoading: '您的数据正在导入中...可能需要几分钟.', + importingRecords: '正在导入 {{count}} 条数据...可能需要几分钟.', statusSuccess: '数据导入成功！', statusError: '数据导入失败！', diff --git a/client/src/i18n/en/insert.ts b/client/src/i18n/en/insert.ts index c583d665..b3b12ecc 100644 --- a/client/src/i18n/en/insert.ts +++ b/client/src/i18n/en/insert.ts @@ -7,10 +7,10 @@ const insertTrans = { sample: 'CSV Sample', noteTitle: 'Note', notes: [ - `CSV or JSON file is supported`, - `Ensure data column names match field label names in Schema.`, - `Data should be <150MB and <100,000 rows for proper import.`, - `"Import File" only appends new records; it doesn't update existing ones.`, + `CSV or JSON file is supported.`, + `Ensure CSV column names or JSON key values match schema field names.`, + `File size should be <256MB.`, + `"Import File" only inserts new data; it doesn't support upsert.`, ], overSizeWarning: 'File data size should less than {{size}}MB', isContainFieldNames: 'First row contains field names?', @@ -25,8 +25,9 @@ const insertTrans = { previewTipAction: '*Change header cell selector value to edit field name', requiredFieldName: 'Field Name*', - statusLoading: 'Your data is importing now...It may take few minutes', - statusLoadingTip: 'Please wait patiently, thank you', + statusLoading: 'Importing your data... This may take a few minutes.', + importingRecords: + 'Importing {{count}} records... This may take a few minutes.', statusSuccess: 'Import File Successfully!', statusError: 'Import File Failed!', diff --git a/client/src/pages/dialogs/ImportSampleDialog.tsx b/client/src/pages/dialogs/ImportSampleDialog.tsx index 0f2354b4..5eb43eb5 100644 --- a/client/src/pages/dialogs/ImportSampleDialog.tsx +++ b/client/src/pages/dialogs/ImportSampleDialog.tsx @@ -33,8 +33,14 @@ const sizeOptions = [ label: '10k', value: '10000', }, + { + label: '100k', + value: '100000', + }, ]; +const biggestSize = Number(sizeOptions[sizeOptions.length - 1].value); + const ImportSampleDialog: FC<{ collection: CollectionObject; cb?: Function; @@ -189,7 +195,7 @@ const ImportSampleDialog: FC<{ value={size} onChange={(event: any, newValue: string | null) => { if (newValue && /^\d+$/.test(newValue)) { - const val = Math.min(Number(newValue), 10000).toString(); + const val = Math.min(Number(newValue), biggestSize).toString(); setSize(val); setCsvFileName( `${collection.collection_name}.sample.${val}.csv` @@ -203,7 +209,7 @@ const ImportSampleDialog: FC<{ if (/^\d*$/.test(newInputValue)) { let val = newInputValue; if (val) { - val = Math.min(Number(val), 10000).toString(); + val = Math.min(Number(val), biggestSize).toString(); } setSize(val); setCsvFileName( diff --git a/client/src/pages/dialogs/insert/Dialog.tsx b/client/src/pages/dialogs/insert/Dialog.tsx index f1ea2d52..92de044c 100644 --- a/client/src/pages/dialogs/insert/Dialog.tsx +++ b/client/src/pages/dialogs/insert/Dialog.tsx @@ -48,6 +48,7 @@ const InsertContainer: FC = ({ InsertStatusEnum.init ); const [insertFailMsg, setInsertFailMsg] = useState(''); + const [importingCount, setImportingCount] = useState(0); const [nextDisabled, setNextDisabled] = useState(false); @@ -308,6 +309,9 @@ const InsertContainer: FC = ({ fields! ); + // Set the number of records being imported + setImportingCount(data.length); + const param: InsertDataParam = { partition_name: partitionValue, fields_data: data, @@ -325,13 +329,9 @@ const InsertContainer: FC = ({ setInsertStatus(InsertStatusEnum.success); } } catch (err: any) { - const { - response: { - data: { message }, - }, - } = err; - setInsertFailMsg(message); - setInsertStatus(InsertStatusEnum.error); + // back to import step + setInsertStatus(InsertStatusEnum.init); + setActiveStep(InsertStepperEnum.import); } }; @@ -409,7 +409,13 @@ const InsertContainer: FC = ({ ); // default represents InsertStepperEnum.status default: - return ; + return ( + + ); } }; diff --git a/client/src/pages/dialogs/insert/Status.tsx b/client/src/pages/dialogs/insert/Status.tsx index 7c8b2777..2c80c0e6 100644 --- a/client/src/pages/dialogs/insert/Status.tsx +++ b/client/src/pages/dialogs/insert/Status.tsx @@ -5,30 +5,41 @@ import type { InsertStatusProps } from './Types'; import successPath from '@/assets/imgs/insert/success.png'; import failPath from '@/assets/imgs/insert/fail.png'; import { InsertStatusEnum } from './consts'; - import Box from '@mui/material/Box'; -const InsertStatus: FC = ({ status, failMsg }) => { +const InsertStatus: FC = ({ status, failMsg, importingCount }) => { const { t: insertTrans } = useTranslation('insert'); - const textSx = (theme: Theme) => ({ marginTop: theme.spacing(3) }); - const loadingTipSx = (theme: Theme) => ({ marginBottom: theme.spacing(6) }); - const loadingSvgSx = (theme: Theme) => ({ - color: theme.palette.primary.main, + const iconSx = { + width: '64px', + height: '64px', + marginBottom: 2, + }; + + const textSx = (theme: Theme) => ({ + marginTop: theme.spacing(2), + textAlign: 'center', + }); + + const errorTextSx = (theme: Theme) => ({ + ...textSx(theme), + color: theme.palette.error.main, }); + const wrapperSx = (theme: Theme) => ({ - width: '75vw', - height: status === InsertStatusEnum.loading ? '288px' : '200px', + width: '40vw', + minHeight: '200px', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', + padding: theme.spacing(4), }); const InsertSuccess = () => ( <> - insert success - + insert success + {insertTrans('statusSuccess')} @@ -36,25 +47,24 @@ const InsertStatus: FC = ({ status, failMsg }) => { const InsertLoading = () => ( <> - - - {insertTrans('statusLoading')} - - ({ ...textSx(theme), ...loadingTipSx(theme) })} - > - {insertTrans('statusLoadingTip')} + + + {insertTrans('importingRecords', { count: importingCount })} ); + const InsertError = () => ( <> - insert error - + insert error + {insertTrans('statusError')} - {failMsg && {failMsg}} + {failMsg && ( + + {failMsg} + + )} ); @@ -64,7 +74,6 @@ const InsertStatus: FC = ({ status, failMsg }) => { return ; case InsertStatusEnum.success: return ; - // status error or init as default default: return ; } diff --git a/client/src/pages/dialogs/insert/Types.ts b/client/src/pages/dialogs/insert/Types.ts index 1de4518b..65bf8df4 100644 --- a/client/src/pages/dialogs/insert/Types.ts +++ b/client/src/pages/dialogs/insert/Types.ts @@ -61,4 +61,5 @@ export interface InsertPreviewProps { export interface InsertStatusProps { status: InsertStatusEnum; failMsg: string; + importingCount: number; } diff --git a/server/src/app.ts b/server/src/app.ts index cacfd5d6..913edfdd 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -71,7 +71,7 @@ app.use( }) ); // limit json file size -app.use(express.json({ limit: '150MB' })); +app.use(express.json({ limit: '256MB' })); // TransformResInterceptor app.use(TransformResMiddleware); // LoggingInterceptor diff --git a/server/src/collections/collections.service.ts b/server/src/collections/collections.service.ts index ec954f7e..5bbc8344 100644 --- a/server/src/collections/collections.service.ts +++ b/server/src/collections/collections.service.ts @@ -30,6 +30,7 @@ import { LoadState, AlterCollectionFieldPropertiesReq, AlterIndexReq, + ErrorCode, } from '@zilliz/milvus2-sdk-node'; import { Parser } from '@json2csv/plainjs'; import { @@ -284,8 +285,30 @@ export class CollectionsService { async insert(clientId: string, data: InsertReq) { const { milvusClient } = clientCache.get(clientId); - const res = await milvusClient.insert(data); - return res; + const BATCH_SIZE = 1000; + const fields_data = data.fields_data || data.data; + + // If data size is less than or equal to batch size, insert directly + if (!fields_data || fields_data.length <= BATCH_SIZE) { + const res = await milvusClient.insert(data); + return res; + } + + // Handle insertion in batches + const results = []; + for (let i = 0; i < fields_data.length; i += BATCH_SIZE) { + const batchData = fields_data.slice(i, i + BATCH_SIZE); + // Create a new request with only the necessary fields + const batchRequest = { ...data, fields_data: batchData } as InsertReq; + const result = await milvusClient.insert(batchRequest); + + if (result.status.error_code !== ErrorCode.SUCCESS) { + throw new Error(result.status.reason); + } + results.push(result); + } + + return results[0]; } async upsert(clientId: string, data: InsertReq) { @@ -613,7 +636,6 @@ export class CollectionsService { db_name, }); - const BATCH_SIZE = 1000; const totalSize = parseInt(size, 10); const fields_data = genRows( collectionInfo.schema.fields, @@ -630,31 +652,12 @@ export class CollectionsService { // If download is true, return the generated data directly return { sampleFile }; } else { - // Handle insertion in batches if size > 1000 - if (totalSize <= BATCH_SIZE) { - return await this.insert(clientId, { - collection_name, - fields_data, - db_name, - }); - } - - const results = []; - for (let i = 0; i < totalSize; i += BATCH_SIZE) { - const batchData = fields_data.slice(i, i + BATCH_SIZE); - const result = await this.insert(clientId, { - collection_name, - fields_data: batchData, - db_name, - }); - results.push(result); - } - - return { - status: 'success', - message: `Successfully inserted ${totalSize} records in ${Math.ceil(totalSize / BATCH_SIZE)} batches`, - results, - }; + // Insert all data at once, batch handling is now in insert method + return await this.insert(clientId, { + collection_name, + fields_data, + db_name, + }); } }