SMW rebuildData: better logging and attempt error handling

enterprisemediawiki · Oct 18, 2018 · a8e3486 · a8e3486
1 parent 115cfa8
commit a8e3486
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 5 deletions.
diff --git a/src/playbooks/rebuild-smw-and-index.yml b/src/playbooks/rebuild-smw-and-index.yml
@@ -17,7 +17,7 @@
     - search-index
 
   - name: Rebuild SemanticMediaWiki data for each wiki
-    shell: "bash {{ m_deploy }}/smw-rebuild-all.sh"
+    shell: "bash {{ m_deploy }}/smw-rebuild-all.sh > {{ m_logs }}/smw-rebuilddata/smw-rebuild-all.log"
     run_once: true
     tags:
     - smw-data

diff --git a/src/roles/create-wiki-wrapper/tasks/main.yml b/src/roles/create-wiki-wrapper/tasks/main.yml
@@ -39,7 +39,7 @@
   - search-index
 
 - name: Rebuild SemanticMediaWiki data for each wiki
-  shell: "bash {{ m_deploy }}/smw-rebuild-all.sh {{ wiki_id }}"
+  shell: "bash {{ m_deploy }}/smw-rebuild-all.sh {{ wiki_id }} > {{ m_logs }}/smw-rebuilddata/smw-rebuild-all.log"
   run_once: true
   tags:
   - smw-data
diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml
@@ -475,7 +475,7 @@
   run_once: true
 
 - name: "(Re-)build SemanticMediaWiki data for: {{ wikis_to_rebuild_data | join(', ') }}"
-  shell: "bash {{ m_deploy }}/smw-rebuild-all.sh \"{{ wikis_to_rebuild_data | join(' ') }}\""
+  shell: "bash {{ m_deploy }}/smw-rebuild-all.sh \"{{ wikis_to_rebuild_data | join(' ') }}\"  > {{ m_logs }}/smw-rebuilddata/smw-rebuild-all.log"
   when: (docker_skip_tasks is not defined or not docker_skip_tasks) and (wikis_to_rebuild_data|length > 0)
   run_once: true
   tags:

diff --git a/src/roles/mediawiki/templates/smw-rebuild-all.sh.j2 b/src/roles/mediawiki/templates/smw-rebuild-all.sh.j2
@@ -46,6 +46,8 @@ for d in $do_wikis; do
 
 	while [ "$start_id" != "-1" ]; do
 
+		previous_start_id="$start_id"
+
 		WIKI="$wiki_id" \
 		php "{{ m_mediawiki }}/extensions/SemanticMediaWiki/maintenance/rebuildData.php" \
 		-d 5 -v $dependent_option --ignore-exceptions \
@@ -55,12 +57,25 @@ for d in $do_wikis; do
 		--exception-log="$exception_log" \
 		>> "$out_log" 2>&1
 
+		# Set a new start id
+		start_id=$(cat "$start_id_file")
+
+		# If something happens to prevent start_id from incrementing, manually increment it.
+		if [ "$previous_start_id" = "$start_id" ]; then
+			last_id_logged=$(grep "... updating" "$out_log" | tail -1 | awk '{print $3}')
+			next_id=$((last_id_logged+1))
+			echo "Start ID not updating. Was $start_id. Last ID logged = $last_id_logged. Incrementing to $next_id" >> "$exception_log"
+			start_id="$next_id"
+		fi
+
 		# after first run, don't rebuild structure, but do skip rebuilding properties
 		dependent_option="--skip-properties"
 
-		start_id=$(cat "$start_id_file")
-
 		echo "next pass start id = $start_id"
+		echo "$(date +"%F_%T")"
+
+		echo "next pass start id = $start_id" >> "$out_log"
+		echo "$(date +"%F_%T")" >> "$out_log"
 
 	done