dicom-anonymiser Generic Wrapper Microservice

.gitignore

@@ -321,3 +321,6 @@ coverage/
 htmlcov/
 src/plugin/SmiPlugin/nuget.exe
 rdmp-cli/
+
+# exclude anonymiser output data
+/src/microservices/Microservices.DicomAnonymiser/Development/AnonymiserData/extractRoot

src/applications/SRAnonTool/CTP_SRAnonTool.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+# CTP_SRAnonTool.sh
+#   This script is called from CTP via the SRAnonTool config in default.yaml.
+# It is called with -i DICOMfile -o OutputDICOMfile
+# It extracts the text from DICOMfile, passes it through the SemEHR anonymiser,
+# and reconstructs the structured text into OutputDICOMfile.
+# NOTE: the -o file must already exist! because only the text part is updated.
+# NOTE: semehr needs python2 but all other tools need python3.
+# XXX TODO: copy the input to the output if it doesn't exist?
+
+prog=$(basename "$0")
+progdir=$(dirname "$0")
+usage="usage: ${prog} [-d] [-v] [-e virtualenv] [-s semehr_root] [-y yaml] -i read_from.dcm  -o write_into.dcm"
+options="dve:s:y:i:o:"
+semehr_dir="/opt/semehr"
+virtenv=""
+debug=0
+verbose=0
+
+# Default value if not set is for SMI
+if [ "$SMI_ROOT" == "" ]; then
+	echo "${prog}: WARNING: env var SMI_ROOT was not set, using default value" >&2
+	export SMI_ROOT="/nfs/smi/home/smi"
+fi
+if [ "$SMI_LOGS_ROOT" == "" ]; then
+	echo "${prog}: WARNING: env var SMI_LOGS_ROOT was not set, using default value" >&2
+	export SMI_LOGS_ROOT="/beegfs-hdruk/smi/data/logs"
+fi
+
+# Configure logging
+log="$SMI_LOGS_ROOT/${prog}/${prog}.log"
+mkdir -p `dirname "${log}"`
+touch $log
+echo "`date` $@" >> $log
+
+# Error reporting and exit
+tidy_exit()
+{
+	rc=$1
+	msg="$2"
+	echo "$msg" >&2
+	echo "`date` $msg" >> $log
+	# Tidy up, if not debugging
+	if [ $debug -eq 0 ]; then
+	  if [ -f "${input_doc}" ]; then rm -f "${input_doc}"; fi
+	  if [ -f "${anon_doc}" ]; then rm -f "${anon_doc}"; fi
+	  if [ -f "${anon_xml}" ]; then rm -f "${anon_xml}"; fi
+	  # Prefer not to use rm -fr for safety
+	  if [ -d "${semehr_input_dir}" ]; then rm -f "${semehr_input_dir}/"*; fi
+	  if [ -d "${semehr_input_dir}" ]; then rmdir "${semehr_input_dir}"; fi
+	  if [ -d "${semehr_output_dir}" ]; then rm -f "${semehr_output_dir}/"*; fi
+	  if [ -d "${semehr_output_dir}" ]; then rmdir "${semehr_output_dir}"; fi
+	fi
+	# Tell user where log file is when failure occurs
+	if [ $rc -ne 0 ]; then echo "See log file $log" >&2; fi
+	exit $rc
+}
+
+# Default executable PATHs and Python libraries
+export PATH=${PATH}:${SMI_ROOT}/bin:${SMI_ROOT}/scripts:${progdir}
+if [ "$PYTHONPATH" == "" ]; then
+	export PYTHONPATH=${SMI_ROOT}/lib/python3:${SMI_ROOT}/lib/python3/virtualenvs/semehr/$(hostname -s)/lib/python3.6/site-packages:${SMI_ROOT}/lib/python3/virtualenvs/semehr/$(hostname -s)/lib64/python3.6/site-packages
+fi
+
+# Command line arguments
+while getopts ${options} var; do
+case $var in
+	d) debug=1;;
+	v) verbose=1;;
+	e) virtenv="$OPTARG";;
+	y) default_yaml0="$OPTARG";;
+	i) input_dcm="$OPTARG";;
+	o) output_dcm="$OPTARG";;
+	s) semehr_dir="$OPTARG";;
+	?) echo "$usage" >&2; exit 1;;
+esac
+done
+shift $(($OPTIND - 1))
+
+if [ ! -f "$input_dcm" ]; then
+	tidy_exit 2 "ERROR: cannot read input file '${input_dcm}'"
+fi
+if [ ! -f "$output_dcm" ]; then
+	#tidy_exit 3 "ERROR: cannot write to ${output_dcm} because it must already exist"
+	cp "$input_dcm" "$output_dcm"
+	chmod +w "$output_dcm"
+fi
+
+# Activate the virtual environment
+if [ "$virtenv" != "" ]; then
+	if [ -f "$virtenv/bin/activate" ]; then
+		source "$virtenv/bin/activate"
+	else
+		echo "ERROR: Cannot activate virtual environment ${virtenv} - no bin/activate script" >&2
+		exit 1
+	fi
+fi
+
+# Find the config files, if not specified try SMI defaults otherwise in the repo
+if [ "$default_yaml0" == "" ]; then
+	if [ -f "$SMI_ROOT/configs/smi_dataExtract.yaml" ]; then
+		default_yaml0="$SMI_ROOT/configs/smi_dataLoad_mysql.yaml"
+		default_yaml1="$SMI_ROOT/configs/smi_dataExtract.yaml"
+	else
+		default_yaml0="${progdir}/../../../data/microserviceConfigs/default.yaml"
+	fi
+fi
+if [ "$default_yaml1" == "" ]; then
+	default_yaml1="$default_yaml0"
+fi
+
+
+# ---------------------------------------------------------------------
+# Determine the SemEHR filenames - create per-process directories
+semehr_input_dir=$(mktemp  -d -t input_docs.XXXX --tmpdir=${semehr_dir}/data)
+semehr_output_dir=$(mktemp -d -t anonymised.XXXX --tmpdir=${semehr_dir}/data)
+if [ "$semehr_input_dir" == "" ]; then
+	tidy_exit 8 "Cannot create temporary directory in ${semehr_dir}/data"
+fi
+if [ "$semehr_output_dir" == "" ]; then
+	tidy_exit 9 "Cannot create temporary directory in ${semehr_dir}/data"
+fi
+
+doc_filename=$(basename "$input_dcm")
+input_doc="${semehr_input_dir}/${doc_filename}"
+anon_doc="${semehr_output_dir}/${doc_filename}"
+anon_xml="${semehr_output_dir}/${doc_filename}.knowtator.xml"
+
+# ---------------------------------------------------------------------
+# Convert DICOM to text
+#  Reads  $input_dcm
+#  Writes $input_doc
+if [ $verbose -gt 0 ]; then
+	echo "RUN: CTP_DicomToText.py  -y $default_yaml0 -y $default_yaml1 -i ${input_dcm} -o ${input_dcm}.SRtext"
+fi
+CTP_DicomToText.py  -y $default_yaml0 -y $default_yaml1 \
+	-i "${input_dcm}" \
+	-o "${input_doc}"  || tidy_exit 4 "Error $? from CTP_DicomToText.py while converting ${input_dcm} to ${input_doc}"
+
+# ---------------------------------------------------------------------
+# Run the SemEHR anonymiser using a set of private directories
+#  Reads  $input_doc
+#  Writes $anon_doc, and $anon_xml via the --xml option
+#
+semehr_anon.py -s "${semehr_dir}" -i "${input_doc}" -o "${anon_doc}" --xml || tidy_exit 5 "Error running SemEHR-anon given ${input_doc} from ${input_dcm}"
+# If there's still no XML file then exit
+if [ ! -f "$anon_xml" ]; then
+	tidy_exit 6 "ERROR: SemEHR-anon failed to convert $input_doc to $anon_xml"
+fi
+
+# ---------------------------------------------------------------------
+# Convert XML back to DICOM
+#  Reads  $input_dcm and $anon_xml
+#  Writes $output_dcm (must already exist)
+if [ $verbose -gt 0 ]; then
+	echo "RUN: CTP_XMLToDicom.py -y $default_yaml1 	-i $input_dcm -x $anon_xml -o $output_dcm"
+fi
+CTP_XMLToDicom.py -y $default_yaml1 \
+	-i "$input_dcm" \
+	-x "$anon_xml" \
+	-o "$output_dcm"   || tidy_exit 7 "Error $? from CTP_XMLToDicom.py while redacting $output_dcm with $anon_xml"
+
+tidy_exit 0 "Finished with ${input_dcm}"

src/applications/SRAnonTool/semehr_anon.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# Anonymise a text file or a directory of text files
+# Usage: semehr_anon.sh -i input_dir -o output_dir
+#    or: semehr_anon.sh -i input_file -o output_file
+#          -s path to the parent of CogStack-SemEHR dir.
+#          --spacy to use SpaCy named entity recogniser.
+#          --xml to write .knowtator.xml files.
+# To anonymise *DICOM* files you need CTP_SRAnonTool.sh
+# (which uses CTP_DicomToText, this script, and CTP_XMLToDicom).
+# NOTE: this script has superseded semehr_anon.sh.
+
+import argparse, json, logging, re, sys, os, glob
+import shutil # for copyfile
+import tempfile # for TemporaryDirectory
+import subprocess # for run
+from SmiServices import Knowtator
+from logging import handlers
+
+# Configuration
+use_spacy = False
+empty_knowtator_xml_document_string = '<?xml version="1.0" ?>\n<annotations>\n</annotations>\n'
+
+
+def anonymise_dir(input_dir, output_dir, semehr_dir, semehr_anon_cfg_file, write_xml = False):
+
+    input_dir = os.path.abspath(input_dir)
+    output_dir = os.path.abspath(output_dir)
+
+    cfg_file = os.path.join(output_dir, 'anonymisation_task.json')
+    phi_file = os.path.join(output_dir, 'anonymiser.phi')
+    log_file = os.path.join(output_dir, 'anonymiser.log')
+
+    # Create a config file in the output directory
+    with open(semehr_anon_cfg_file) as fd:
+        cfg_json = json.load(fd)
+    cfg_json['text_data_path'] = input_dir
+    cfg_json['anonymisation_output'] = output_dir
+    cfg_json['extracted_phi'] = phi_file
+    cfg_json['grouped_phi_output'] = '/dev/null'
+    cfg_json['logging_file'] = log_file
+    cfg_json['annotation_mode'] = False
+    #cfg_json['number_threads'] = 0  # a bug in CPython causes hang/deadlock trying to acquire lock in logging __init__.py ?
+    cfg_json['use_spacy'] = use_spacy
+    with open(cfg_file, 'w') as fd:
+        print(json.dumps(cfg_json), file=fd)
+
+    # Run SemEHR anonymiser
+    cur_dir = os.getcwd()
+    os.chdir(os.path.join(semehr_dir, 'CogStack-SemEHR', 'anonymisation'))
+    logging.info('Running anonymiser from %s to %s' % (input_dir, output_dir))
+    result = subprocess.run(['python3', 'anonymiser.py', cfg_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    os.chdir(cur_dir)
+    if result.returncode:
+        logging.error('ERROR: SemEHR anonymiser failed: "%s"' % (result.stdout.decode('utf-8') + result.stderr.decode('utf-8')))
+        # XXX should return early?
+
+    # Read the JSON file "phi" to redact the words
+    with open(phi_file) as fd:
+        phi_json = json.load(fd)
+
+    # Collect the set of document names
+    # Can either use the list of input files (if you want every input file to have an output file)
+    # or the list of documents which were redacted (if you don't want failed docs in the output dir)
+    list_of_docs = set([ii['doc'] for ii in phi_json]) # list of anonymised documents
+    if write_xml:
+        list_of_docs = next(os.walk(input_dir))[2]     # list of original filenames
+    logging.info('Redacting %d files in %s' % (len(list_of_docs), output_dir))
+
+    # For each document, reads the output document and fully anonymise it using the 'phi' file
+    # (The output document is almost anonymised but not necessarily fully as extra annotations are in the phi file.
+    # Compared to the input document, the output document also has had the headers removed).
+    for doc in list_of_docs:
+        renamed_dict = [{'start_char':ii['start'], 'end_char':ii['start']+len(ii['sent']), 'text':ii['sent']} for ii in phi_json if ii['doc'] == doc]
+        renamed_dict = sorted(renamed_dict, key = lambda x: x['start_char'])
+        xml_string = Knowtator.dict_to_annotation_xml_string(renamed_dict) if len(renamed_dict) else empty_knowtator_xml_document_string
+
+        # If there's no record of doc in phi then it wasn't anonymised
+        # so we can either copy it (might contain PII!) or create an empty one from /dev/null
+        if not renamed_dict:
+            input_file = '/dev/null' # '/dev/null' OR os.path.join(input_dir, doc)
+        else:
+            input_file = os.path.join(output_dir, doc)
+        with open(input_file, 'r') as fd:
+            text = fd.read()
+
+        for annot in renamed_dict:
+            if annot['end_char'] > annot['start_char']:
+                text = text[:annot['start_char']] + 'X'.rjust(annot['end_char']-annot['start_char'],'X') + text[annot['end_char']:]
+
+        output_file = os.path.join(output_dir, doc)
+        logging.debug('Redacting %s' % doc)
+        with open(output_file, 'w') as fd:
+            fd.write(text)
+        # Write the XML file
+        output_file += '.knowtator.xml'
+        if write_xml:
+            with open(output_file, 'w') as fd:
+                fd.write(xml_string)
+
+    # Tidy up
+    os.remove(cfg_file)
+    os.remove(phi_file)
+    os.remove(log_file)
+    return
+
+
+def anonymise_file(input_file, output_file, semehr_dir, semehr_anon_cfg_file, write_xml = False):
+    # The anonymiser works on whole directories so
+    # create temporary input/output directories and copy the file there.
+    input_file = os.path.abspath(input_file)
+    output_file = os.path.abspath(output_file)
+    input_dir = tempfile.TemporaryDirectory()
+    output_dir = tempfile.TemporaryDirectory()
+    shutil.copyfile(input_file, os.path.join(input_dir.name, os.path.basename(input_file)))
+    anonymise_dir(input_dir.name, output_dir.name, semehr_dir, semehr_anon_cfg_file, write_xml=write_xml)
+    shutil.copyfile(os.path.join(output_dir.name, os.path.basename(input_file)), output_file)
+    if write_xml:
+        xml_src = os.path.join(output_dir.name, os.path.basename(input_file) + '.knowtator.xml')
+        xml_dest = output_file + '.knowtator.xml'
+        if os.path.isfile(xml_src):
+            shutil.copyfile(xml_src, xml_dest)
+    input_dir.cleanup()
+    output_dir.cleanup()
+
+
+def main():
+    global use_spacy
+
+	# Configure logging
+    if not os.environ.get('SMI_LOGS_ROOT'): raise Exception('Environment variable SMI_LOGS_ROOT must be set')
+    log_dir = os.path.expandvars('$SMI_LOGS_ROOT')
+    log_file = os.path.join(log_dir, os.path.basename(sys.argv[0]) + '.log')
+    file_handler = logging.handlers.RotatingFileHandler(filename=log_file, maxBytes=256*1024*1024, backupCount=9)
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    handlers = [file_handler, stdout_handler]
+    logging.basicConfig(level=logging.DEBUG, handlers=handlers,
+        format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s')
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description='Redact text given knowtator XML')
+    parser.add_argument('-i', dest='input', action="store", help='directory of text, or filename of one text file')
+    parser.add_argument('-o', dest='output', action="store", help='path to output filename or directory where redacted text files will be written')
+    parser.add_argument('--xml', dest='write_xml', action="store_true", help='write knowtator.xml in output directory for all input files')
+    parser.add_argument('-s', dest='semehr_dir', action="store", help='path to parent of CogStack-SemEHR directory')
+    parser.add_argument('--spacy', dest='spacy', action='store_true', help='use SpaCy to identify names')
+    args = parser.parse_args()
+    if not args.input:
+        parser.print_help()
+        exit(1)
+
+    if args.spacy:
+    	use_spacy = True # XXX global
+
+    # Default SemEHR directory for testing
+    semehr_dir = '/Users/daniyalarshad/EPCC/github/NationalSafeHaven'
+
+    if os.path.isdir(os.path.expandvars("$HOME/SemEHR")):
+        semehr_dir = os.path.expandvars("$HOME/SemEHR")
+    if os.path.isdir('/opt/semehr'):
+        semehr_dir = '/opt/semehr'
+    if args.semehr_dir:
+        semehr_dir = args.semehr_dir
+    semehr_anon_cfg_file = os.path.join(semehr_dir, 'CogStack-SemEHR', 'anonymisation', 'conf', 'anonymisation_task.json') # i.e. /opt/semehr/CogStack-SemEHR/anonymisation/conf/anonymisation_task.json
+
+    if os.path.isfile(args.input):
+        anonymise_file(args.input, args.output, semehr_dir, semehr_anon_cfg_file, write_xml=args.write_xml)
+
+    elif os.path.isdir(args.input):
+        anonymise_dir(args.input, args.output, semehr_dir, semehr_anon_cfg_file, write_xml=args.write_xml)
+
+
+if __name__ == '__main__':
+    main()

src/common/Smi.Common/Options/GlobalOptions.cs

@@ -487,7 +487,12 @@ public class DicomAnonymiserOptions : IOptions
         public string? RoutingKeySuccess { get; set; }
         public string? RoutingKeyFailure { get; set; }
         public bool FailIfSourceWriteable { get; set; } = true;
-
+        public string? VirtualEnvPath { get; set; }
+        public string? DicomPixelAnonPath { get; set; }
+        public string? SmiServicesPath { get; set; }
+        public string? CtpAnonCliJar { get; set; }
+        public string? CtpAllowlistScript { get; set; }
+        public string? SRAnonymiserToolPath { get; set; }
 
         public override string ToString() => GlobalOptions.GenerateToString(this);
     }

src/microservices/Microservices.DicomAnonymiser/Anonymisers/AnonymiserFactory.cs

@@ -5,14 +5,15 @@
 {
     public static class AnonymiserFactory
     {
-        public static IDicomAnonymiser CreateAnonymiser(DicomAnonymiserOptions dicomAnonymiserOptions)
+        public static IDicomAnonymiser CreateAnonymiser(GlobalOptions options)
         {
-            var anonymiserTypeStr = dicomAnonymiserOptions.AnonymiserType;
+            var anonymiserTypeStr = options.DicomAnonymiserOptions!.AnonymiserType;
             if (!Enum.TryParse(anonymiserTypeStr, ignoreCase: true, out AnonymiserType anonymiserType))
                 throw new ArgumentException($"Could not parse '{anonymiserTypeStr}' to a valid AnonymiserType");
 
             return anonymiserType switch
             {
+                AnonymiserType.DefaultAnonymiser => new DefaultAnonymiser(options),
                 // TODO(rkm 2021-12-07) Can remove the LGTM ignore once an AnonymiserType is implemented
                 _ => throw new NotImplementedException($"No case for AnonymiserType '{anonymiserType}'"), // lgtm[cs/constant-condition]
             };

src/microservices/Microservices.DicomAnonymiser/Anonymisers/AnonymiserType.cs

@@ -6,5 +6,6 @@ public enum AnonymiserType
         /// Unused placeholder value
         /// </summary>
         None = 0,
+        DefaultAnonymiser = 1,
     }
 }