Skip to content

feat: Add export all rows of a class and export in JSON format #2361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jan 25, 2023
207 changes: 136 additions & 71 deletions src/dashboard/Data/Browser/Browser.react.js
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ class Browser extends DashboardView {
filters: new List(),
ordering: '-createdAt',
selection: {},
exporting: false,
exportingCount: 0,

data: null,
lastMax: -1,
Expand Down Expand Up @@ -1255,15 +1257,12 @@ class Browser extends DashboardView {
});
}

async confirmExportSelectedRows(rows) {
this.setState({ rowsToExport: null });
async confirmExportSelectedRows(rows, type, indentation) {
this.setState({ rowsToExport: null, exporting: true, exportingCount: 0 });
const className = this.props.params.className;
const query = new Parse.Query(className);

if (rows['*']) {
// Export all
query.limit(10000);
} else {
if (!rows['*']) {
// Export selected
const objectIds = [];
for (const objectId in this.state.rowsToExport) {
Expand All @@ -1273,75 +1272,136 @@ class Browser extends DashboardView {
query.limit(objectIds.length);
}

const classColumns = this.getClassColumns(className, false);
// create object with classColumns as property keys needed for ColumnPreferences.getOrder function
const columnsObject = {};
classColumns.forEach((column) => {
columnsObject[column.name] = column;
});
// get ordered list of class columns
const columns = ColumnPreferences.getOrder(
columnsObject,
this.context.applicationId,
className
).filter(column => column.visible);
const processObjects = (objects) => {
const classColumns = this.getClassColumns(className, false);
// create object with classColumns as property keys needed for ColumnPreferences.getOrder function
const columnsObject = {};
classColumns.forEach((column) => {
columnsObject[column.name] = column;
});
// get ordered list of class columns
const columns = ColumnPreferences.getOrder(
columnsObject,
this.context.applicationId,
className
).filter((column) => column.visible);

if (type === '.json') {
const element = document.createElement('a');
const file = new Blob(
[
JSON.stringify(
objects.map((obj) => {
const json = obj._toFullJSON();
delete json.__type;
return json;
}),
null,
indentation ? 2 : null,
),
],
{ type: 'application/json' }
);
element.href = URL.createObjectURL(file);
element.download = `${className}.json`;
document.body.appendChild(element); // Required for this to work in FireFox
element.click();
document.body.removeChild(element);
return;
}

const objects = await query.find({ useMasterKey: true });
let csvString = columns.map(column => column.name).join(',') + '\n';
for (const object of objects) {
const row = columns.map(column => {
const type = columnsObject[column.name].type;
if (column.name === 'objectId') {
return object.id;
} else if (type === 'Relation' || type === 'Pointer') {
if (object.get(column.name)) {
return object.get(column.name).id
} else {
return ''
}
} else {
let colValue;
if (column.name === 'ACL') {
colValue = object.getACL();
} else {
colValue = object.get(column.name);
}
// Stringify objects and arrays
if (Object.prototype.toString.call(colValue) === '[object Object]' || Object.prototype.toString.call(colValue) === '[object Array]') {
colValue = JSON.stringify(colValue);
}
if(typeof colValue === 'string') {
if (colValue.includes('"')) {
// Has quote in data, escape and quote
// If the value contains both a quote and delimiter, adding quotes and escaping will take care of both scenarios
colValue = colValue.split('"').join('""');
return `"${colValue}"`;
} else if (colValue.includes(',')) {
// Has delimiter in data, surround with quote (which the value doesn't already contain)
return `"${colValue}"`;
let csvString = columns.map((column) => column.name).join(',') + '\n';
for (const object of objects) {
const row = columns
.map((column) => {
const type = columnsObject[column.name].type;
if (column.name === 'objectId') {
return object.id;
} else if (type === 'Relation' || type === 'Pointer') {
if (object.get(column.name)) {
return object.get(column.name).id;
} else {
return '';
}
} else {
// No quote or delimiter, just include plainly
return `${colValue}`;
let colValue;
if (column.name === 'ACL') {
colValue = object.getACL();
} else {
colValue = object.get(column.name);
}
// Stringify objects and arrays
if (
Object.prototype.toString.call(colValue) ===
'[object Object]' ||
Object.prototype.toString.call(colValue) === '[object Array]'
) {
colValue = JSON.stringify(colValue);
}
if (typeof colValue === 'string') {
if (colValue.includes('"')) {
// Has quote in data, escape and quote
// If the value contains both a quote and delimiter, adding quotes and escaping will take care of both scenarios
colValue = colValue.split('"').join('""');
return `"${colValue}"`;
} else if (colValue.includes(',')) {
// Has delimiter in data, surround with quote (which the value doesn't already contain)
return `"${colValue}"`;
} else {
// No quote or delimiter, just include plainly
return `${colValue}`;
}
} else if (colValue === undefined) {
// Export as empty CSV field
return '';
} else {
return `${colValue}`;
}
}
} else if (colValue === undefined) {
// Export as empty CSV field
return '';
} else {
return `${colValue}`;
})
.join(',');
csvString += row + '\n';
}

// Deliver to browser to download file
const element = document.createElement('a');
const file = new Blob([csvString], { type: 'text/csv' });
element.href = URL.createObjectURL(file);
element.download = `${className}.csv`;
document.body.appendChild(element); // Required for this to work in FireFox
element.click();
document.body.removeChild(element);
};

if (!rows['*']) {
const objects = await query.find({ useMasterKey: true });
processObjects(objects);
this.setState({ exporting: false, exportingCount: objects.length });
} else {
let batch = [];
query.eachBatch(
(obj) => {
batch.push(...obj);
if (batch.length % 10 === 0) {
this.setState({ exportingCount: batch.length });
}
}
}).join(',');
csvString += row + '\n';
const one_gigabyte = Math.pow(2, 30);
const size =
new TextEncoder().encode(JSON.stringify(batch)).length /
one_gigabyte;
if (size.length > 1) {
processObjects(batch);
batch = [];
}
if (obj.length !== 100) {
processObjects(batch);
batch = [];
this.setState({ exporting: false, exportingCount: 0 });
}
},
{ useMasterKey: true }
);
}

// Deliver to browser to download file
const element = document.createElement('a');
const file = new Blob([csvString], { type: 'text/csv' });
element.href = URL.createObjectURL(file);
element.download = `${className}.csv`;
document.body.appendChild(element); // Required for this to work in FireFox
element.click();
document.body.removeChild(element);
}

getClassRelationColumns(className) {
Expand Down Expand Up @@ -1755,7 +1815,7 @@ class Browser extends DashboardView {
className={className}
selection={this.state.rowsToExport}
onCancel={this.cancelExportSelectedRows}
onConfirm={() => this.confirmExportSelectedRows(this.state.rowsToExport)}
onConfirm={(type, indentation) => this.confirmExportSelectedRows(this.state.rowsToExport, type, indentation)}
/>
);
}
Expand All @@ -1772,6 +1832,11 @@ class Browser extends DashboardView {
<Notification note={this.state.lastNote} isErrorNote={false}/>
);
}
else if (this.state.exporting) {
notification = (
<Notification note={`Exporting ${this.state.exportingCount}+ objects...`} isErrorNote={false}/>
);
}
return (
<div>
<Helmet>
Expand Down
32 changes: 26 additions & 6 deletions src/dashboard/Data/Browser/ExportSelectedRowsDialog.react.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,22 @@
* This source code is licensed under the license found in the LICENSE file in
* the root directory of this source tree.
*/
import Modal from 'components/Modal/Modal.react';
import React from 'react';
import Modal from 'components/Modal/Modal.react';
import React from 'react';
import Dropdown from 'components/Dropdown/Dropdown.react';
import Field from 'components/Field/Field.react';
import Label from 'components/Label/Label.react';
import Option from 'components/Dropdown/Option.react';
import Toggle from 'components/Toggle/Toggle.react';

export default class ExportSelectedRowsDialog extends React.Component {
constructor() {
super();

this.state = {
confirmation: ''
confirmation: '',
exportType: '.csv',
indentation: true,
};
}

Expand All @@ -28,13 +35,26 @@ export default class ExportSelectedRowsDialog extends React.Component {
type={Modal.Types.INFO}
icon='warn-outline'
title={this.props.selection['*'] ? 'Export all rows?' : (selectionLength === 1 ? 'Export 1 selected row?' : `Export ${selectionLength} selected rows?`)}
subtitle={this.props.selection['*'] ? 'Note: Exporting is limited to the first 10,000 rows.' : ''}
subtitle={this.props.selection['*'] ? 'Large datasets are exported as multiple files of up to 1 GB each.' : ''}
disabled={!this.valid()}
confirmText='Export'
cancelText='Cancel'
onCancel={this.props.onCancel}
onConfirm={this.props.onConfirm}>
{}
onConfirm={() => this.props.onConfirm(this.state.exportType, this.state.indentation)}>
<Field
label={<Label text='Select export type' />}
input={
<Dropdown
value={this.state.exportType}
onChange={(exportType) => this.setState({ exportType })}>
<Option value='.csv'>.csv</Option>
<Option value='.json'>.json</Option>
</Dropdown>
} />
{this.state.exportType === '.json' && <Field
label={<Label text='Indentation' />}
input={<Toggle value={this.state.indentation} type={Toggle.Types.YES_NO} onChange={(indentation) => {this.setState({indentation})}} />} />
}
</Modal>
);
}
Expand Down