diff --git a/docs/arcgis.mmd b/docs/arcgis.mmd new file mode 100644 index 00000000..8b1ad558 --- /dev/null +++ b/docs/arcgis.mmd @@ -0,0 +1,112 @@ +flowchart LR + + %% Algorithm + gather_stage ==> fetch_stage + fetch_stage ==> import_stage + + subgraph gather_stage [Gather Stage] + direction TB + gs([GATHER STARTED]) + ge([GATHER ENDED]) + gs ==> is_extra_search_criteria + is_extra_search_criteria == Yes ==> add_to_query + is_extra_search_criteria == No ==>basic_query + add_to_query ==> basic_query + basic_query ==> get_for_all_time + get_for_all_time ==> query_arcgis + query_arcgis ==> get_current_objects + get_current_objects ==> compute_new + compute_new ==> create_object + compute_new ==> compute_deleted + compute_deleted ==> create_object + compute_deleted ==> compute_changed + compute_changed ==> is_date_different + is_date_different == Yes ==> create_object + is_date_different == No ==> skip + compute_changed ==> ge + end + subgraph fetch_stage [Fetch Stage] + direction TB + fs([FETCH STARTED]) + fe([FETCH ENDED]) + fs ==> do_nothing + do_nothing ==> fe + end + subgraph import_stage [Import Stage] + direction TB + is([IMPORT STARTED]) + ie([IMPORT ENDED]) + is ==> is_object_empty + is_object_empty-. Yes .-> skip_2 + is_object_empty == No ==> get_existing_object + get_existing_object ==> is_existing_object + is_existing_object == Yes ==> mark_not_current + is_existing_object == No ==> is_delete + mark_not_current ==> is_delete + is_delete == Yes ==> delete + delete ==> is_object_content_empty + is_object_content_empty-. Yes .-> error + is_object_content_empty == No ==> make_package_dict + %% Code: https://github.com/GSA/ckanext-geodatagov/blob/984dc47087f981c15f7878bef5a96970adb78125/ckanext/geodatagov/harvesters/arcgis.py#L338-L431 + make_package_dict-. error .-> ie + make_package_dict ==> is_status_new + is_status_new == Yes ==> default_create_package_schema + is_status_new == No ==> default_update_package_schema + default_update_package_schema ==> is_status_new_2 + default_create_package_schema ==> is_status_new_2 + is_status_new_2 == Yes ==> generate_guid + generate_guid ==> save_object_reference + save_object_reference ==> create + is_status_new_2 == No ==> is_status_changed + is_status_changed == Yes ==> is_existing_object_2 + is_existing_object_2 == Yes ==> mark_not_current + is_existing_object_2 == No ==> update + mark_not_current ==> update + update ==> ie + create ==> ie + is_status_changed == No ==> ie + end + + %% Data + error[\Error/] + skip[/Skip\] + skip_2[/Skip\] + + %% Functons + %% Code: https://github.com/ckan/ckan/blob/master/ckan/logic/schema.py#L115-L194 + default_update_package_schema[[Default Update]] + default_create_package_schema[[Default Create]] + + create_object[[Create New Object]] + update[[Update Dataset]] + do_nothing[[Nothing to do]] + create[[Create New Package]] + delete[[Delete Package]] + save_object_reference[[Save Object Reference in Package]] + generate_guid[[Generate GUID]] + get_existing_object[[Get Existing Harvest Object]] + mark_not_current[[Mark Previous Harvest Object as not current]] + add_to_query[[Add search to basic query]] + basic_query[[Query All data from all times]] + get_for_all_time[[Build data 100 rows at a time]] + query_arcgis[[Query Server]] + get_current_objects[[Get Existing Harvest Objects]] + compute_new[[Calculate new objects]] + compute_deleted[[Calculate deleted objects]] + compute_changed[[Calculate changed objects]] + + %% Code: https://github.com/GSA/ckanext-geodatagov/blob/984dc47087f981c15f7878bef5a96970adb78125/ckanext/geodatagov/harvesters/arcgis.py#L338-L431 + make_package_dict[[ArcGIS Package Create]] + + + %% Conditional Checks + is_extra_search_criteria{Are there extra search parameters?} + is_existing_object{Does the object exist?} + is_existing_object_2{Does the object exist?} + is_object_empty{Is Object Empty?} + is_object_content_empty{Is the Object content empty?} + is_delete{Should the dataset be deleted?} + is_status_new{Is the Status new?} + is_status_new_2{Is the Status new?} + is_status_changed{Is the Status changed?} + is_date_different{Is the Date different?} diff --git a/docs/arcgis.svg b/docs/arcgis.svg new file mode 100644 index 00000000..2f6ab748 --- /dev/null +++ b/docs/arcgis.svg @@ -0,0 +1 @@ +
Import Stage
Yes
No
Yes
No
Yes
Yes
No
error
Yes
No
Yes
No
Yes
Yes
No
No
IMPORT STARTED
Is Object Empty?
Skip
Get Existing Harvest Object
Does the object exist?
Mark Previous Harvest Object as not current
Should the dataset be deleted?
Delete Package
Is the Object content empty?
Error
ArcGIS Package Create
IMPORT ENDED
Is the Status new?
Default Create
Default Update
Is the Status new?
Generate GUID
Save Object Reference in Package
Create New Package
Is the Status changed?
Does the object exist?
Update Dataset
Fetch Stage
FETCH STARTED
Nothing to do
FETCH ENDED
Gather Stage
Yes
No
Yes
No
GATHER STARTED
Are there extra search parameters?
Add search to basic query
Query All data from all times
Build data 100 rows at a time
Query Server
Get Existing Harvest Objects
Calculate new objects
Create New Object
Calculate deleted objects
Calculate changed objects
Is the Date different?
Skip
GATHER ENDED
 \ No newline at end of file