Merge 22.7 to develop

snprc_ehr/build.gradle

@@ -13,4 +13,9 @@ dependencies
             BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:ehrModules:ehr", depProjectConfig: "published", depExtension: "module")
             BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:dataintegration", depProjectConfig: "published", depExtension: "module")
             BuildUtils.addLabKeyDependency(project: project, config: "modules", depProjectPath: ":server:modules:snd", depProjectConfig: "published", depExtension: "module")
+
+            external "org.apache.commons:commons-text:1.9"
+            compileOnly "org.projectlombok:lombok:1.18.22"
+            annotationProcessor "org.projectlombok:lombok:1.18.22"
+
         }

snprc_ehr/resources/credits/dependencies.txt

@@ -0,0 +1,2 @@
+# direct external dependencies for project :server:modules:snprcEHRModules:snprc_ehr
+commons-text-1.9.jar

snprc_ehr/resources/credits/jars.txt

@@ -0,0 +1,4 @@
+{table}
+Filename|Component|Version|Source|License|SNPRC Dev|Purpose
+commons-text-1.9.jar|Apache Commons Text|1.9|{link:GitHub|https://github.com/apache/commons-text}|{link:Apache License 2.0|https://github.com/apache/commons-text/blob/master/LICENSE.txt}|jallen|A library focused on algorithms working on strings
+{table}

snprc_ehr/resources/data/reports.tsv

@@ -53,7 +53,6 @@ idHistory	General	query	ID History	true		study	idHistory			id_date	false	false
 generalTb	General	query	TB Test Results	true		study	tb			date	false	false			TB Test results
 demographics	General	query	Demographics	true		study	demographics	By Location			false	false		qcstate/publicdata	This report displays the demographics data about each animal including species, gender and birth
 reproSummary	Reproductive Management	js	Reproductive Summary	true		study	underDevelopment			date	false	false		qcstate/publicdata
-menses	Reproductive Management	js	Menses	true		study	underDevelopment			date	false	false		qcstate/publicdata
 freezerworks	General	query	Freezer/Sample Data	true		study	Freezerworks			date	false	false		qcstate/publicdata
 currentBlood	General	js	Current Blood	true		study	currentBlood			date	false	false			This report contains a summary of the current available blood for each animal
 Obscan	Reproductive Management	query	Obscan	true		study	ObScan			date	false	false		qcstate/publicdata

snprc_ehr/resources/queries/ehr_lookups/ageclass.query.xml

@@ -15,7 +15,7 @@
                 <fk>
                   <fkDbSchema>snprc_ehr</fkDbSchema>
                   <fkTable>species</fkTable>
-                  <fkColumnName>arc_species_code.code</fkColumnName>
+                  <fkColumnName>arc_species_code</fkColumnName>
                 </fk>
               </column>
               <column columnName="ageclass">

snprc_ehr/resources/queries/study/CycleDaily/.qview.xml

@@ -13,7 +13,7 @@
         </column>
         <column name="VaginalBleeding">
             <properties>
-                <property name = "columnTitle" value="VI"/>
+                <property name = "columnTitle" value="VB"/>
             </properties>
         </column>
         <column name="PurpleColor">

snprc_ehr/resources/queries/study/PotentialSires.sql

@@ -21,7 +21,9 @@ FROM study.demographics AS d
            gender,
            label
     FROM ehr_lookups.ageclass AS ac
-) as x ON d.species.arc_species_code = x.species and x.label = 'Adult' AND (x.gender = 'M' OR x.gender is NULL)
+) as x ON d.species.arc_species_code = x.species
+    AND (x.label = 'Adult' OR (d.species.arc_species_code = 'CJ' AND x.label = 'Senior' ))
+    AND (x.gender = 'M' OR x.gender is NULL)
 
 -- hard code until the gestation data is available in LK - use 185 if we don't have a value
          INNER JOIN (

snprc_ehr/src/org/labkey/snprc_ehr/constants/QueryConstants.java

@@ -0,0 +1,120 @@
+package org.labkey.snprc_ehr.constants;
+
+import org.apache.commons.text.CaseUtils;
+
+/**
+ * String constants used for database queries. Contains table names, schema names, column names, SQL aueries,
+ * and other database query information
+ */
+public class QueryConstants
+{
+    public static final String AGE_AT_TIME_COLUMN = "Age At Time";
+    public static final String AGE_AT_TIME_YEARS_COLUMN = "Age At Time Years";
+    public static final String AGE_AT_TIME_YEARS_ROUNDED_COLUMN = "Age At Time Years Rounded";
+    public static final String AGE_AT_TIME_DAYS_COLUMN = "Age At Time Days";
+    public static final String AGE_AT_TIME_MONTHS_COLUMN = "Age At Time Months";
+    public static final String AGE_CLASS_AT_TIME_COLUMN = "Age Class At Time";
+    public static final String ASSIGNMENT_AT_TIME_COLUMN = "Assignment At Time";
+    public static final String STUDY_SCHEMA = "study";
+    public static final String CODE_COLUMN = "code";
+    public static final String SPECIES_TABLE = "species";
+    public static final String PROJECT_TABLE = "project";
+    public static final String ANIMAL_TABLE = "animal";
+    public static final String ASSIGNMENT_TABLE = "assignment";
+    public static final String DATE_COLUMN = "date";
+    public static final String DEMOGRAPHICS_TABLE = "demographics";
+    public static final String ID_COLUMN = "Id";
+    public static final String ROW_ID_COLUMN = "rowid";
+    public static final String PROTOCOLS_AT_TIME_CALCULATED = "Protocols At Time";
+    public static final String ACCOUNTS_AT_TIME_CALCULATED = "Accounts At Time";
+    public static final String PRIMARY_KEY_VARIABLE = "primaryKeyColumnName";
+    public static final String SCHEMA_VARIABLE = "schemaName";
+    public static final String EHR_PATH_VARIABLE = "ehrPath";
+    public static final String QUERY_VARIABLE = "queryName";
+    public static final String ID_COLUMN_VARIABLE = "idColumnName";
+    public static final String TARGET_CONTAINER_VARIABLE ="targetContainerName";
+    public static final String DATE_COLUMN_VARIABLE = "dateColumnName";
+    public static final String EHR_LOOKUPS_SCHEMA = "ehr_lookups";
+    public static final String AGE_CLASS_LOOKUP_URL = "/detailsQueryRow.view?schemaName=ehr_lookups&query.queryName=ageclass&rowid=";
+
+
+    public static final String AGE_CLASS_TABLE = "ageClass";
+    public static final String ASSIGNMENT_AT_TIME_SQL = "SELECT\n" +
+            "sd.${" + PRIMARY_KEY_VARIABLE + "},\n" +
+            "group_concat(DISTINCT h.protocol.displayName, chr(10)) as " + CaseUtils.toCamelCase(PROTOCOLS_AT_TIME_CALCULATED, true) + ",\n" +
+            "group_concat(DISTINCT ao.account.account, chr(10)) as " + CaseUtils.toCamelCase(ACCOUNTS_AT_TIME_CALCULATED, true) + ",\n" +
+            "FROM \"${" + SCHEMA_VARIABLE + "}\".\"${" + QUERY_VARIABLE + "}\" sd\n" +
+            "JOIN \"${" + TARGET_CONTAINER_VARIABLE + "}\".study.assignment h\n" +
+            "  ON (sd.id = h.id AND h.dateOnly <= CAST(sd.${" + DATE_COLUMN_VARIABLE + "} AS DATE) AND (CAST(sd.${" + DATE_COLUMN_VARIABLE +"} AS DATE) <= h.enddateCoalesced) AND h.qcstate.publicdata = true)\n" +
+            "JOIN \"${" + TARGET_CONTAINER_VARIABLE + "}\".study.animalAccounts ao\n" +
+            "  ON (sd.id = ao.id AND ao.dateOnly <= CAST(sd.${" + DATE_COLUMN_VARIABLE + "} AS DATE) AND (CAST(sd.${" + DATE_COLUMN_VARIABLE +"} AS DATE) <= ao.enddateCoalesced) AND ao.qcstate.publicdata = true)\n" +
+            "group by sd.${"+ PRIMARY_KEY_VARIABLE + "}";
+
+    public static final String AGE_AT_TIME_SQL = "SELECT\n" +
+            "c.${" + PRIMARY_KEY_VARIABLE + "},\n" +
+            "\n" +
+            "CAST(\n" +
+            "CASE\n" +
+            "WHEN d.birth is null or c.${" + DATE_COLUMN_VARIABLE + "} is null\n" +
+            "  THEN null\n" +
+            "WHEN (d.lastDayAtCenter IS NOT NULL AND d.lastDayAtCenter < c.${" + DATE_COLUMN_VARIABLE + "}) THEN\n" +
+            " ROUND(CONVERT(age_in_months(d.birth, d.lastDayAtCenter), DOUBLE) / 12, 1)\n" +
+            "ELSE\n" +
+            "  ROUND(CONVERT(age_in_months(d.birth, CAST(c.${" + DATE_COLUMN_VARIABLE + "} as DATE)), DOUBLE) / 12, 1)\n" +
+            "END AS float) as " + CaseUtils.toCamelCase(AGE_AT_TIME_COLUMN, true) + ",\n" +
+            "\n" +
+
+            "CAST(\n" +
+            "CASE\n" +
+            "WHEN d.birth is null or c.${" + DATE_COLUMN_VARIABLE + "} is null\n" +
+            "  THEN null\n" +
+            "WHEN (d.lastDayAtCenter IS NOT NULL AND d.lastDayAtCenter < c.${" + DATE_COLUMN_VARIABLE + "}) THEN\n" +
+            " ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, d.lastDayAtCenter), DOUBLE) / 365.25, 2)\n" +
+            "ELSE\n" +
+            "  ROUND(CONVERT(timestampdiff('SQL_TSI_DAY', d.birth, CAST(c.${" + DATE_COLUMN_VARIABLE + "} as DATE)), DOUBLE) / 365.25, 2)\n" +
+            "END AS float) as " + CaseUtils.toCamelCase(AGE_AT_TIME_YEARS_COLUMN, true) + ",\n" +
+            "\n" +
+            "CAST(\n" +
+            "CASE\n" +
+            "WHEN d.birth is null or c.${" + DATE_COLUMN_VARIABLE + "} is null\n" +
+            "  THEN null\n" +
+            "WHEN (d.lastDayAtCenter IS NOT NULL AND d.lastDayAtCenter < c.${" + DATE_COLUMN_VARIABLE + "}) THEN\n" +
+            " floor(age(d.birth, d.lastDayAtCenter))\n" +
+            "ELSE\n" +
+            "  floor(age(d.birth, CAST(c.${" + DATE_COLUMN_VARIABLE + "} as DATE)))\n" +
+            "END AS float) as " + CaseUtils.toCamelCase(AGE_AT_TIME_YEARS_ROUNDED_COLUMN, true) + ",\n" +
+            "\n" +
+            //Added 'Age at time Days' by kollil on 02/15/2019
+            "CAST(\n" +
+            "CASE\n" +
+            "WHEN d.birth is null or c.${" + DATE_COLUMN_VARIABLE + "} is null\n" +
+            "  THEN null\n" +
+            "WHEN (d.lastDayAtCenter IS NOT NULL AND d.lastDayAtCenter < c.${" + DATE_COLUMN_VARIABLE + "}) THEN\n" +
+            "  CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, d.lastDayAtCenter), INTEGER)\n" +
+            "ELSE\n" +
+            "  CONVERT(TIMESTAMPDIFF('SQL_TSI_DAY',d.birth, CAST(c.${" + DATE_COLUMN_VARIABLE + "} AS DATE)), INTEGER)\n" +
+            "END AS float) as " + CaseUtils.toCamelCase(AGE_AT_TIME_DAYS_COLUMN, true) + ",\n" +
+            "\n" +
+            //
+            "CAST(\n" +
+            "CASE\n" +
+            "WHEN d.birth is null or c.${" + DATE_COLUMN_VARIABLE + "} is null\n" +
+            "  THEN null\n" +
+            "WHEN (d.lastDayAtCenter IS NOT NULL AND d.lastDayAtCenter < c.${" + DATE_COLUMN_VARIABLE + "}) THEN\n" +
+            "  CONVERT(age_in_months(d.birth, d.lastDayAtCenter), INTEGER)\n" +
+            "ELSE\n" +
+            "  CONVERT(age_in_months(d.birth, CAST(c.${" + DATE_COLUMN_VARIABLE + "} AS DATE)), INTEGER)\n" +
+            "END AS float) as " + CaseUtils.toCamelCase(AGE_AT_TIME_MONTHS_COLUMN, true) + ",\n" +
+            //NOTE: written as subselect so we ensure a single row returned in case data in ehr_lookups.ageclass has rows that allow dupes
+            "(SELECT ac.label FROM ehr_lookups.ageclass ac\n" +
+            "  WHERE " +
+            "  (CONVERT(age_in_months(d.birth, COALESCE(d.lastDayAtCenter, now())), DOUBLE) / 12) >= ac.\"min\" AND\n" +
+            "  ((CONVERT(age_in_months(d.birth, COALESCE(d.lastDayAtCenter, now())), DOUBLE) / 12) < ac.\"max\" OR ac.\"max\" is null) AND\n" +
+            "  d.species.arc_species_code = ac.species AND\n" +
+            "  (d.gender = ac.gender OR ac.gender IS NULL)\n" +
+            ") AS " + CaseUtils.toCamelCase(AGE_CLASS_AT_TIME_COLUMN, true) + " \n" +
+            "FROM \"${" + SCHEMA_VARIABLE + "}\".\"${" + QUERY_VARIABLE + "}\" c " +
+            "LEFT JOIN \"${" + EHR_PATH_VARIABLE + "}\".study.demographics d ON (d.Id = c.${" + ID_COLUMN_VARIABLE + "})";
+
+
+}

snprc_ehr/src/org/labkey/snprc_ehr/helpers/CustomizerQueryProvider.java

@@ -0,0 +1,171 @@
+package org.labkey.snprc_ehr.helpers;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrSubstitutor;
+import org.apache.commons.text.CaseUtils;
+import org.labkey.api.data.AbstractTableInfo;
+import org.labkey.api.data.ColumnInfo;
+import org.labkey.api.data.TableInfo;
+import org.labkey.api.data.WrappedColumn;
+import org.labkey.api.query.UserSchema;
+import org.labkey.snprc_ehr.model.CalculatedColumn;
+import org.labkey.snprc_ehr.model.CalculatedColumnQueryInfo;
+import org.labkey.snprc_ehr.query.CalculatedColumnForeignKey;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.labkey.snprc_ehr.constants.QueryConstants.DATE_COLUMN_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.EHR_PATH_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.ID_COLUMN;
+import static org.labkey.snprc_ehr.constants.QueryConstants.ID_COLUMN_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.PRIMARY_KEY_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.QUERY_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.SCHEMA_VARIABLE;
+import static org.labkey.snprc_ehr.constants.QueryConstants.TARGET_CONTAINER_VARIABLE;
+
+/**
+ * Class that provides methods to build a table from a SQL query that contains columns to be calculated
+ */
+public class CustomizerQueryProvider
+{
+    /**
+     * Constructor
+     */
+    public CustomizerQueryProvider() {
+
+    }
+
+    /**
+     * Checks if the passed table exists, the primary key and id columns,
+     * and builds table with calculated fields from SQL query. Returns false if any checks fail and
+     * true if all checks pass and the table is successfully built
+     * @param tableInfo
+     * @param columnName
+     * @param dateColumnName
+     * @param queryString
+     * @param ehrSchema
+     * @param calculatedColumns
+     * @param isRemovingDefaultCustomizerColumns
+     * @return
+     */
+    public AbstractTableInfo getCalculatedColumnsTable(AbstractTableInfo tableInfo, String columnName, String dateColumnName,
+                                       String queryString, UserSchema ehrSchema,
+                                       Set<CalculatedColumn> calculatedColumns, boolean isRemovingDefaultCustomizerColumns) {
+        /* Check if the column already exists in the table */
+        if (tableInfo.getColumn(CaseUtils.toCamelCase(columnName, false), false) != null)
+        {
+            /* Check if the table should be removed so that it can be rebuilt. Applies to tables that require
+            *  queries that are different from the DefaultEHRCustomizer's query */
+            if (isRemovingDefaultCustomizerColumns)
+                tableInfo.removeColumn(tableInfo.getColumn(CaseUtils.toCamelCase(columnName, false)));
+            else
+                return null;
+        }
+
+        final ColumnInfo primaryKeyColumn = getPrimaryKeyColumn(tableInfo);
+        if (primaryKeyColumn == null)
+            return null;
+        final ColumnInfo idColumn = tableInfo.getColumn(ID_COLUMN);
+        if(idColumn == null)
+            return null;
+
+        /* Build a query info object to be used by the foreign key class */
+        CalculatedColumnQueryInfo queryInfo = getQueryInfo(tableInfo, primaryKeyColumn, idColumn, ehrSchema, columnName,
+                calculatedColumns);
+
+        /* Build a wrapped column object for the table */
+        WrappedColumn calculatedColumn = getWrappedCalculatedColumn(queryInfo, mapQueryStringValues(queryString, queryInfo,
+                dateColumnName));
+        tableInfo.addColumn(calculatedColumn);
+        return tableInfo;
+    }
+
+    /**
+     * Obtain the column info for the primary key of a given table
+     * @param tableInfo
+     * @return
+     */
+    private ColumnInfo getPrimaryKeyColumn(TableInfo tableInfo) {
+        List<ColumnInfo> primaryKeys = tableInfo.getPkColumns();
+        return (primaryKeys.size() != 1) ? null : primaryKeys.get(0);
+    }
+
+    /**
+     * Returns a new query info object to be used for calculating a column's info
+     * @param tableInfo
+     * @param primaryKeyColumn
+     * @param idColumn
+     * @param ehrSchema
+     * @param label
+     * @param calculatedColumns
+     * @return
+     */
+    private CalculatedColumnQueryInfo getQueryInfo(AbstractTableInfo tableInfo, ColumnInfo primaryKeyColumn,
+                                                   ColumnInfo idColumn, UserSchema ehrSchema,
+                                                   String label, Set<CalculatedColumn> calculatedColumns)
+    {
+        CalculatedColumnQueryInfo queryInfo = new CalculatedColumnQueryInfo();
+        queryInfo.setTableInfo(tableInfo);
+        queryInfo.setCalculatedColumns(calculatedColumns);
+        queryInfo.setPrimaryKeyColumn(primaryKeyColumn);
+        queryInfo.setIdColumn(idColumn);
+        queryInfo.setEhrSchema(ehrSchema);
+        queryInfo.setLabel(label);
+
+        return queryInfo;
+    }
+
+    /**
+     * Returns a wrapped column object from a given query and it's information
+     * @param queryInfo
+     * @param queryString
+     * @return
+     */
+    private WrappedColumn getWrappedCalculatedColumn(CalculatedColumnQueryInfo queryInfo, String queryString) {
+        WrappedColumn column = new WrappedColumn(queryInfo.getPrimaryKeyColumn(), CaseUtils.toCamelCase(queryInfo.getLabel(),
+                false));
+        column.setLabel(queryInfo.getLabel());
+        column.setReadOnly(true);
+        column.setIsUnselectable(true);
+        column.setUserEditable(false);
+        /* Invokes the CalculatedColumnForeignKey class */
+        column.setFk(new CalculatedColumnForeignKey(queryInfo, queryString));
+        return column;
+    }
+
+    /**
+     * Maps values from the query object's data to the variable names within the SQL query string
+     * (formatted for the Apache Commons Lang StrSubstitor where ${variableName} maps to variableName)
+     * @param queryString
+     * @param queryInfo
+     * @param dateColumnName
+     * @return
+     */
+    private String mapQueryStringValues(String queryString, CalculatedColumnQueryInfo queryInfo, String dateColumnName) {
+        Map<String, String> queryByValues = new HashMap<>();
+        if(StringUtils.contains(queryString, PRIMARY_KEY_VARIABLE))
+            queryByValues.put(PRIMARY_KEY_VARIABLE, queryInfo.getPrimaryKeyColumn().getFieldKey().toSQLString());
+        if(StringUtils.contains(queryString, SCHEMA_VARIABLE))
+            queryByValues.put(SCHEMA_VARIABLE, queryInfo.getTableInfo().getPublicSchemaName());
+        if(StringUtils.contains(queryString, QUERY_VARIABLE))
+            queryByValues.put(QUERY_VARIABLE, queryInfo.getTableInfo().getName());
+        if(StringUtils.contains(queryString, EHR_PATH_VARIABLE))
+            queryByValues.put(EHR_PATH_VARIABLE, queryInfo.getEhrSchema().getContainer().getPath());
+        if(StringUtils.contains(queryString, ID_COLUMN_VARIABLE))
+            queryByValues.put(ID_COLUMN_VARIABLE, queryInfo.getIdColumn().getFieldKey().toSQLString());
+        if(StringUtils.contains(queryString, TARGET_CONTAINER_VARIABLE))
+            queryByValues.put(TARGET_CONTAINER_VARIABLE, queryInfo.getTableInfo().getUserSchema().getName());
+        if(StringUtils.contains(queryString, DATE_COLUMN_VARIABLE))
+            queryByValues.put(DATE_COLUMN_VARIABLE, dateColumnName);
+        return StrSubstitutor.replace(queryString, queryByValues);
+
+    }
+
+
+
+
+
+}