Skip to content

Commit

Permalink
gpkg progress (#1125)
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidQuartz
DavidQuartz authored Jul 29, 2022
1 parent cb60073 commit d00a9f7
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 29 deletions.
32 changes: 30 additions & 2 deletions geonode_mapstore_client/client/js/api/geonode/v2/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ let endpoints = {
'keywords': '/api/v2/keywords',
'regions': '/api/v2/regions',
'groups': '/api/v2/groups',
'uploads': '/api/v2/uploads'
'uploads': '/api/v2/uploads',
'status': '/api/v2/resource-service/execution-status',
'exectionRequest': '/api/v2/executionrequest'
};

const RESOURCES = 'resources';
Expand All @@ -59,6 +61,8 @@ const CATEGORIES = 'categories';
const KEYWORDS = 'keywords';
const GROUPS = 'groups';
const UPLOADS = 'uploads';
const STATUS = 'status';
const EXECUTIONREQUEST = 'exectionRequest';

function addCountToLabel(name, count) {
return `${name} (${count || 0})`;
Expand Down Expand Up @@ -772,6 +776,18 @@ export const getPendingUploads = () => {
.then(({ data }) => data?.uploads);
};

export const getPendingExecutionRequests = () => {
return axios.get(parseDevHostname(endpoints[EXECUTIONREQUEST]), {
params: {
'filter{action}': 'import',
'page': 1,
'page_size': 99999
}
})
.then(({ data }) => data?.requests)
.catch(() => null);
};

export const getProcessedUploadsById = (ids) => {
return axios.get(parseDevHostname(endpoints[UPLOADS]), {
params: {
Expand Down Expand Up @@ -831,6 +847,15 @@ export const uploadDocument = ({
.then(({ data }) => (data));
};

export const getExecutionStatus = (executionId) => {
return axios.get(`${parseDevHostname(endpoints[STATUS])}/${executionId}`)
.then(({ data }) => ({...data, id: executionId, create_date: data.created }));
};

export const deleteExecutionRequest = (executionId) => {
return axios.delete(`${parseDevHostname(endpoints[EXECUTIONREQUEST])}/${executionId}`);
};

export default {
getEndpoints,
getResources,
Expand Down Expand Up @@ -864,7 +889,10 @@ export default {
downloadResource,
getDatasets,
getPendingUploads,
getPendingExecutionRequests,
getProcessedUploadsById,
getProcessedUploadsByImportId,
uploadDocument
uploadDocument,
getExecutionStatus,
deleteExecutionRequest
};
27 changes: 17 additions & 10 deletions geonode_mapstore_client/client/js/routes/UploadDataset.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import {
getPendingUploads,
getProcessedUploadsById,
getProcessedUploadsByImportId,
uploadDataset
uploadDataset,
getPendingExecutionRequests,
deleteExecutionRequest
} from '@js/api/geonode/v2';
import axios from '@mapstore/framework/libs/ajax';
import UploadListContainer from '@js/routes/upload/UploadListContainer';
Expand Down Expand Up @@ -171,20 +173,19 @@ function UploadList({
});
}
if (successfulUploads.length > 0) {
const successfulUploadsIds = successfulUploads.map(({ data }) => data?.id);
const successfulUploadsIds = successfulUploads.filter(({ data }) => !!data?.id).map(({data}) => data?.id);
const successfulUploadsNames = successfulUploads.map(({ baseName }) => baseName);
updateWaitingUploads(omit(waitingUploads, successfulUploadsNames));
getProcessedUploadsByImportId(successfulUploadsIds)

successfulUploadsIds.length > 0 && getProcessedUploadsByImportId(successfulUploadsIds)
.then((successfulUploadProcesses) => {
onSuccess(successfulUploadProcesses);
setLoading(false);
})
.catch(() => {
setLoading(false);
});
} else {
setLoading(false);
}
setLoading(false);
})
.catch(() => {
setLoading(false);
Expand Down Expand Up @@ -239,20 +240,23 @@ function ProcessingUploadList({
updatePending.current = () => {
if (!loading) {
setLoading(true);
getPendingUploads()
axios.all([getPendingUploads(), getPendingExecutionRequests()])
.then(incomingUploads => [...incomingUploads[0], ...incomingUploads[1]])
.then((newPendingUploads) => {
if (isMounted.current) {
const failedPendingUploads = pendingUploads.filter(({ state }) => state === 'INVALID');
const newIds = newPendingUploads.map(({ id }) => id);
const pendingImports = newPendingUploads.filter(({ action, exec_id: execitionId }) => !!action && action === 'import' && !deletedIds.includes(execitionId)).map(pendingImport => ({ ...pendingImport, create_date: pendingImport.created, id: pendingImport.exec_id }));
const missingIds = pendingUploads
.filter(upload => (upload.state !== 'PROCESSED' && upload.state !== 'INVALID') && !newIds.includes(upload.id) && !deletedIds.includes(upload.id))
.filter(upload => (!!upload.state && upload.state !== 'PROCESSED' && upload.state !== 'INVALID') && !newIds.includes(upload.id) && !deletedIds.includes(upload.id))
.map(({ id }) => id);
const currentProcessed = pendingUploads.filter((upload) => upload.state === 'PROCESSED');
if (missingIds.length > 0) {
getProcessedUploadsById(missingIds)
.then((processed) => {
onChange([
...failedPendingUploads,
...pendingImports,
...processed,
...currentProcessed,
...newPendingUploads
Expand All @@ -262,6 +266,7 @@ function ProcessingUploadList({
.catch(() => {
onChange([
...failedPendingUploads,
...pendingImports,
...currentProcessed,
...newPendingUploads
]);
Expand All @@ -270,6 +275,7 @@ function ProcessingUploadList({
} else {
onChange([
...failedPendingUploads,
...pendingImports,
...currentProcessed,
...newPendingUploads
]);
Expand All @@ -285,8 +291,9 @@ function ProcessingUploadList({
}
};

function handleDelete({ id, deleteUrl }) {
axios.get(deleteUrl)
function handleDelete({ id, deleteUrl = null }) {
const deleteRequest = deleteUrl ? () => axios.get(deleteUrl) : () => deleteExecutionRequest(id);
deleteRequest()
.finally(() => {
if (isMounted.current) {
setDeletedIds((ids) => [...ids, id]);
Expand Down
21 changes: 14 additions & 7 deletions geonode_mapstore_client/client/js/routes/upload/UploadCard.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ function UploadCard({
resumeUrl,
onRemove,
error,
type
type,
status,
errorLog
}) {

const { datasetMaxUploadSize, documentMaxUploadSize, maxParallelUploads } = getConfigProp('geoNodeSettings') || {};
Expand All @@ -46,7 +48,7 @@ function UploadCard({
return (
<div className="gn-upload-card">
<div className="gn-upload-card-header">
{state === 'INVALID' ? <div className="gn-upload-card-error"><FaIcon name="exclamation"/></div> : null}
{(state === 'INVALID' || status === 'failed') ? <div className="gn-upload-card-error"><FaIcon name="exclamation" /></div> : null}
<div className="gn-upload-card-title">
{detailUrl
? <a
Expand All @@ -58,7 +60,7 @@ function UploadCard({
</a>
: name}
</div>
{(progress < 100 && progress > 0) ? <Spinner /> : null}
{((progress < 100 && progress > 0) || status === 'running') ? <Spinner /> : null}
{onRemove
? <Button size="xs" onClick={onRemove}>
<FaIcon name="trash"/>
Expand All @@ -78,7 +80,7 @@ function UploadCard({
<Message msgId="gnviewer.completeUpload" />
</Button>
: null}
{detailUrl
{(detailUrl || status === 'finished')
? <Button
variant="primary"
href={detailUrl}
Expand All @@ -88,8 +90,12 @@ function UploadCard({
<Message msgId="gnviewer.view" />
</Button>
: null}
{state === 'INVALID'
? <ErrorMessageWithTooltip tooltipId={<Message msgId={`gnviewer.${getUploadErrorMessageFromCode(error?.code)}`} msgParams={{ limit: getUploadErrorMessageFromCode(error?.code) === 'fileExceeds' ? maxAllowedSize : maxParallelUploads }} />} />
{(state === 'INVALID' || status === 'failed')
? <>
{!errorLog ? <ErrorMessageWithTooltip tooltipId={<Message msgId={`gnviewer.${getUploadErrorMessageFromCode(error?.code)}`} msgParams={{ limit: getUploadErrorMessageFromCode(error?.code) === 'fileExceeds' ? maxAllowedSize : maxParallelUploads }} />} />
: <ErrorMessageWithTooltip tooltip={getUploadErrorMessageFromCode(null, errorLog)} />
}
</>
: null}
</div>
</div>
Expand Down Expand Up @@ -117,7 +123,8 @@ UploadCard.defaultProps = {
name: '',
state: '',
progress: 0,
type: 'document'
type: 'document',
status: ''
};

export default UploadCard;
14 changes: 10 additions & 4 deletions geonode_mapstore_client/client/js/routes/upload/UploadListContainer.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,26 +66,32 @@ function UploadListContainer({
name,
progress = 0,
state,
status,
create_date: createDate,
detail_url: detailUrl,
resume_url: resumeUrl,
delete_url: deleteUrl,
error
error,
log,
exec_id: execId,
created
}) => {
return (error !== 'CANCELED' &&
<li
key={id}
key={id || execId}
>
<UploadCard
name={name}
state={state}
detailUrl={detailUrl}
progress={progress}
createDate={createDate}
createDate={createDate || created}
resumeUrl={resumeUrl}
onRemove={deleteUrl ? () => onDelete({ id, deleteUrl }) : null}
onRemove={deleteUrl ? () => onDelete({ id, deleteUrl }) : execId ? () => onDelete({ id: execId }) : null}
error={error}
type={resourceType}
status={status}
errorLog={log}
/>
</li>
);
Expand Down
5 changes: 4 additions & 1 deletion geonode_mapstore_client/client/js/utils/ErrorUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
* LICENSE file in the root directory of this source tree.
*/

export const getUploadErrorMessageFromCode = (code) => {
export const getUploadErrorMessageFromCode = (code, log) => {
if (log) {
return log;
}
switch (code) {
case 'upload_parallelism_limit_exceeded': {
return 'parallelLimitError';
Expand Down
26 changes: 22 additions & 4 deletions geonode_mapstore_client/client/js/utils/ResourceUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,12 +605,30 @@ export const parseUploadResponse = (upload) => {

export const processUploadResponse = (response) => {
const newResponse = response.reduce((acc, currentResponse) => {
const duplicate = acc.find((upload) => upload.id === currentResponse.id);
const duplicate = acc.find((upload) => {
if (upload.id && currentResponse.id) {
return upload.id === currentResponse.id;
} else if (upload.id && currentResponse.exec_id) {
return upload.id === currentResponse.exec_id;
} else if (upload.exec_id && currentResponse.id) {
return upload.exec_id === currentResponse.id;
}
return upload.exec_id === currentResponse.exec_id;
});
if (duplicate) {
const newAcc = acc.filter((upload) => upload.id !== duplicate.id);
return [currentResponse, ...newAcc];
const newAcc = acc.filter((upload) => {
if (upload.id && currentResponse.id) {
return upload.id !== currentResponse.id;
} else if (upload.id && currentResponse.exec_id) {
return upload.id !== currentResponse.exec_id;
} else if (upload.exec_id && currentResponse.id) {
return upload.exec_id !== currentResponse.id;
}
return upload.exec_id !== currentResponse.exec_id;
});
return [{...currentResponse, ...(!currentResponse.id && {create_date: currentResponse.created, id: currentResponse.exec_id})}, ...newAcc];
}
return [currentResponse, ...acc];
return [{...currentResponse, ...(!currentResponse.id && {create_date: currentResponse.created, id: currentResponse.exec_id})}, ...acc];
}, []);

const uploads = parseUploadResponse(newResponse);
Expand Down
19 changes: 18 additions & 1 deletion geonode_mapstore_client/client/js/utils/__tests__/ResourceUtils-test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,25 @@ describe('Test Resource Utils', () => {
state: 'COMPLETE',
progress: 100,
complete: true
},
{
exec_id: 23,
name: 'test3',
created: '2022-05-13T12:24:54.042291Z',
status: 'running',
complete: false
}];

expect(processUploadResponse([...prev, ...current])).toEqual([
{
exec_id: 23,
name: 'test3',
created: '2022-05-13T12:24:54.042291Z',
status: 'running',
complete: false,
create_date: '2022-05-13T12:24:54.042291Z',
id: 23
},
{
id: 1,
name: 'test1',
Expand Down Expand Up @@ -550,7 +566,8 @@ describe('Test Resource Utils', () => {
progress: 40,
complete: false,
resume_url: 'test/upload/delete/439'
}];
}
];

expect(parseUploadResponse(uploads)).toEqual([
{
Expand Down

0 comments on commit d00a9f7

Please sign in to comment.