1409 importer handling of imvalid files

src/OSPSuite.Assets/UIConstants.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Drawing;
 using System.Linq;
 using System.Text;
 using OSPSuite.Assets.Extensions;
+using OSPSuite.Utility.Collections;
 using OSPSuite.Utility.Extensions;
 
 namespace OSPSuite.Assets
@@ -1275,6 +1277,9 @@ public static class Error
       public static readonly string NaNOnData = "Data contains NaN values at imported columns. Select a different action for NaN values or clean your data.";
       public static readonly string UnsupportedFileType = "The type of file that you are trying to open is not currently supported";
       public static readonly string CannotRemoveBaseGridColumnStillInUse = "Cannot remove base grid column still used by other columns";
+      public static readonly string SimpleParseErrorMessage = "There were errors while parsing your data. Navigate to the sheets to read the concrete error.";
+
+      public static string ParseErrorMessage(IEnumerable<string> errors) => $"There were errors while parsing your data: {string.Join(". ", errors)}";
 
       public static string ErrorWhenPlottingDataRepository(int sheetName, string exceptionMessage) => $"It was not possible to plot the data sets. Please, check your configuration for any missing grouping or meta data parameter. An error occur while plotting data set number:{sheetName + 1} produced the following error: {exceptionMessage}";
 

src/OSPSuite.Infrastructure.Import/Core/BaseGridColumnNotFoundException.cs

@@ -0,0 +1,11 @@
+using OSPSuite.Assets;
+
+namespace OSPSuite.Infrastructure.Import.Core
+{
+   public class BaseGridColumnNotFoundException : AbstractImporterException
+   {
+      public BaseGridColumnNotFoundException(string columnName) : base(Error.BaseGridColumnNotFoundException(columnName))
+      {
+      }
+   }
+}

src/OSPSuite.Infrastructure.Import/Core/ColumnNotFoundException.cs

@@ -0,0 +1,11 @@
+using OSPSuite.Assets;
+
+namespace OSPSuite.Infrastructure.Import.Core
+{
+   public class ColumnNotFoundException : AbstractImporterException
+   {
+      public ColumnNotFoundException(string columnName) : base(Error.ColumnNotFound(columnName))
+      {
+      }
+   }
+}

src/OSPSuite.Infrastructure.Import/Core/DataSource.cs

@@ -3,28 +3,63 @@
 using System.Linq;
 using OSPSuite.Core.Domain;
 using OSPSuite.Core.Import;
+using OSPSuite.Infrastructure.Import.Core.Exceptions;
+using OSPSuite.Infrastructure.Import.Core.Helpers;
 using OSPSuite.Infrastructure.Import.Extensions;
 using OSPSuite.Infrastructure.Import.Services;
 using OSPSuite.Utility.Collections;
+using OSPSuite.Utility.Extensions;
 
 namespace OSPSuite.Infrastructure.Import.Core
 {
+   public class ParseErrors
+   {
+      private Cache<IDataSet, List<ParseErrorDescription>> _errors = new Cache<IDataSet, List<ParseErrorDescription>>(onMissingKey: _ => new List<ParseErrorDescription>());
+
+      public bool Any() => _errors.Any();
+
+      public bool Contains(IDataSet key) => _errors.Contains(key);
+
+      public IEnumerable<ParseErrorDescription> ErrorsFor(IDataSet key) => _errors[key];
+
+      public void Add(IDataSet key, ParseErrorDescription x)
+      {
+         Add(key, new List<ParseErrorDescription>() { x });
+      }
+
+      public void Add(ParseErrors other)
+      {
+         foreach (var x in other._errors.KeyValues)
+         {
+            Add(x.Key, x.Value);
+         }
+      }
+
+      public void Add(IDataSet key, IEnumerable<ParseErrorDescription> list)
+      {
+         if (_errors.Contains(key))
+            _errors[key].AddRange(list);
+         else
+            _errors.Add(key, new List<ParseErrorDescription>(list));
+      }
+   }
+
    /// <summary>
    /// Collection of DataSets
    /// </summary>
    public interface IDataSource
    {
       void SetDataFormat(IDataFormat dataFormat);
       void SetNamingConvention(string namingConvention);
-      void AddSheets(Cache<string, DataSheet> dataSheets, IReadOnlyList<ColumnInfo> columnInfos, string filter);
+      ParseErrors AddSheets(Cache<string, DataSheet> dataSheets, IReadOnlyList<ColumnInfo> columnInfos, string filter);
       void SetMappings(string fileName, IEnumerable<MetaDataMappingConverter> mappings);
       ImporterConfiguration GetImporterConfiguration();
       IEnumerable<MetaDataMappingConverter> GetMappings();
       Cache<string, IDataSet> DataSets { get; }
       IEnumerable<string> NamesFromConvention();
       NanSettings NanSettings { get; set; }
       ImportedDataSet DataSetAt(int index);
-      void ValidateDataSourceUnits(IReadOnlyList<ColumnInfo> columnInfos);
+      ParseErrors ValidateDataSourceUnits(IReadOnlyList<ColumnInfo> columnInfos);
    }
 
    public class DataSource : IDataSource
@@ -72,16 +107,18 @@ private Cache<string, DataSheet> filterSheets(Cache<string, DataSheet> dataSheet
          return filteredDataSheets;
       }
 
-      public void AddSheets(Cache<string, DataSheet> dataSheets, IReadOnlyList<ColumnInfo> columnInfos, string filter)
-      {         
+      public ParseErrors AddSheets(Cache<string, DataSheet> dataSheets, IReadOnlyList<ColumnInfo> columnInfos, string filter)
+      {
          _importer.AddFromFile(_configuration.Format, filterSheets(dataSheets, filter), columnInfos, this);
          if (NanSettings == null || !double.TryParse(NanSettings.Indicator, out var indicator))
             indicator = double.NaN;
+         var errors = new ParseErrors();
          foreach (var dataSet in DataSets.KeyValues)
          {
             if (NanSettings != null && NanSettings.Action == NanSettings.ActionType.Throw)
             {
-               dataSet.Value.ThrowsOnNan(indicator);
+               if (dataSet.Value.NanValuesExist(indicator))
+                  errors.Add(dataSet.Value, new NaNParseErrorDescription());
             }
             else
             {
@@ -91,9 +128,10 @@ public void AddSheets(Cache<string, DataSheet> dataSheets, IReadOnlyList<ColumnI
                   continue;
 
                var emptyDataSetsNames = emptyDataSets.Select(d => string.Join(".", d.Description.Where(metaData => metaData.Value != null).Select(metaData => metaData.Value)));
-               throw new EmptyDataSetsException(emptyDataSetsNames);
+               errors.Add(dataSet.Value, new EmptyDataSetsParseErrorDescription(emptyDataSetsNames));
             }
          }
+         return errors;
       }
 
       public void SetMappings(string fileName, IEnumerable<MetaDataMappingConverter> mappings)
@@ -145,8 +183,9 @@ public ImportedDataSet DataSetAt(int index)
       }
 
       //checks that the dimension of all the units coming from columns for error have the same dimension to the corresponding measurement
-      private void validateErrorAgainstMeasurement(IReadOnlyList<ColumnInfo> columnInfos)
+      private ParseErrors validateErrorAgainstMeasurement(IReadOnlyList<ColumnInfo> columnInfos)
       {
+         var errors = new ParseErrors();
          foreach (var column in columnInfos.Where(c => !c.IsAuxiliary()))
          {
             foreach (var relatedColumn in columnInfos.Where(c => c.IsAuxiliary() && c.RelatedColumnOf == column.Name))
@@ -161,7 +200,10 @@ private void validateErrorAgainstMeasurement(IReadOnlyList<ColumnInfo> columnInf
                         continue;
 
                      if (errorColumn.Value != null && measurementColumn.Value.Count != errorColumn.Value.Count)
-                        throw new MismatchingArrayLengthsException();
+                     {
+                        errors.Add(dataSet, new MismatchingArrayLengthsParseErrorDescription());
+                        continue;
+                     }
 
                      var errorDimension = errorColumn.Key.Column.Dimension;
                      var measurementDimension = measurementColumn.Key.Column.Dimension;
@@ -176,7 +218,10 @@ private void validateErrorAgainstMeasurement(IReadOnlyList<ColumnInfo> columnInf
                            var measurementSupportedDimension = column.SupportedDimensions.FirstOrDefault(x => x.HasUnit(measurementColumn.Value.ElementAt(i).Unit));
                            var errorSupportedDimension = column.SupportedDimensions.FirstOrDefault(x => x.HasUnit(errorColumn.Value.ElementAt(i).Unit));
                            if (measurementSupportedDimension != errorSupportedDimension)
-                              throw new ErrorUnitException();
+                           {
+                              errors.Add(dataSet, new ErrorUnitParseErrorDescription());
+                              continue;
+                           }
                         }
                      }
                      else
@@ -188,17 +233,22 @@ private void validateErrorAgainstMeasurement(IReadOnlyList<ColumnInfo> columnInf
                            continue;
 
                         if (measurementDimension != errorDimension)
-                           throw new ErrorUnitException();
+                        {
+                           errors.Add(dataSet, new ErrorUnitParseErrorDescription());
+                           continue;
+                        }
                      }
                   }
                }
             }
          }
+         return errors;
       }
       //checks that all units coming from a mapped column unit belong to a valid dimension for this mapping
       //and also that they are all of the same dimension within every data set. 
-      private void validateUnitsSupportedAndSameDimension(IReadOnlyList<ColumnInfo> columnInfos)
+      private ParseErrors validateUnitsSupportedAndSameDimension(IReadOnlyList<ColumnInfo> columnInfos)
       {
+         var errors = new ParseErrors();
          foreach (var columnInfo in columnInfos)
          {
             foreach (var dataSet in DataSets)
@@ -227,22 +277,30 @@ private void validateUnitsSupportedAndSameDimension(IReadOnlyList<ColumnInfo> co
 
                         //if the unit specified does not belong to one of the supported dimensions of the mapping
                         if (dimension == null)
-                           throw new InvalidDimensionException(currentValue.Unit, columnInfo.DisplayName);
+                        {
+                           errors.Add(dataSet, new InvalidDimensionParseErrorDescription(currentValue.Unit, columnInfo.DisplayName));
+                           continue;
+                        }
 
                         //if the unit specified is not of the same dimension as the other units of the same data set
                         if (dimension != dimensionOfFirstUnit)
-                           throw new InconsistentDimensionBetweenUnitsException(columnInfo.DisplayName);
+                        {
+                           errors.Add(dataSet, new InconsistentDimensionBetweenUnitsParseErrorDescription(columnInfo.DisplayName));
+                           continue;
+                        }
                      }
                   }
                }
             }
          }
+         return errors;
       }
 
-      void IDataSource.ValidateDataSourceUnits(IReadOnlyList<ColumnInfo> columnInfos)
+      ParseErrors IDataSource.ValidateDataSourceUnits(IReadOnlyList<ColumnInfo> columnInfos)
       {
-         validateUnitsSupportedAndSameDimension(columnInfos);
-         validateErrorAgainstMeasurement(columnInfos);
+         var errors = validateUnitsSupportedAndSameDimension(columnInfos);
+         errors.Add(validateErrorAgainstMeasurement(columnInfos));
+         return errors;
       }
    }
 

src/OSPSuite.Infrastructure.Import/Core/DataSourceFile.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using OSPSuite.Infrastructure.Import.Services;
 using OSPSuite.Utility.Collections;
 
@@ -23,7 +24,16 @@ public abstract class DataSourceFile : IDataSourceFile
 
       public IDataFormat Format { get; set; }
 
-      public IList<IDataFormat> AvailableFormats { get; set; }
+      private IList<IDataFormat> _availableFormats;
+      public IList<IDataFormat> AvailableFormats 
+      {
+         get => _availableFormats; 
+         set
+         {
+            _availableFormats = value;
+            Format = value.FirstOrDefault();
+         }
+      }
 
       protected DataSourceFile(IImportLogger logger)
       {

src/OSPSuite.Infrastructure.Import/Core/EmptyNamingConventionsException.cs

@@ -0,0 +1,11 @@
+using OSPSuite.Assets;
+
+namespace OSPSuite.Infrastructure.Import.Core
+{
+   public class EmptyNamingConventionsException : AbstractImporterException
+   {
+      public EmptyNamingConventionsException() : base(Error.NamingConventionEmpty)
+      {
+      }
+   }
+}

src/OSPSuite.Infrastructure.Import/Core/Exceptions/ParseErrorDescription.cs

@@ -0,0 +1,67 @@
+using OSPSuite.Assets;
+using OSPSuite.Utility.Extensions;
+using System.Collections.Generic;
+
+namespace OSPSuite.Infrastructure.Import.Core.Exceptions
+{
+   public abstract class ParseErrorDescription
+   {
+      public string Message { get; protected set; }
+   }
+
+   public class InvalidDimensionParseErrorDescription : ParseErrorDescription
+   {
+      public InvalidDimensionParseErrorDescription(string invalidUnit, string mappingName)
+      {
+         Message = Error.InvalidDimensionException(invalidUnit, mappingName);
+      }
+   }
+
+   public class NaNParseErrorDescription : ParseErrorDescription
+   {
+      public NaNParseErrorDescription()
+      {
+         Message = Error.NaNOnData;
+      }
+   }
+
+   public class MismatchingArrayLengthsParseErrorDescription : ParseErrorDescription
+   {
+      public MismatchingArrayLengthsParseErrorDescription()
+      {
+         Message = Error.MismatchingArrayLengths;
+      }
+   }
+
+   public class InvalidMappingColumnParseErrorDescription : ParseErrorDescription
+   {
+      public InvalidMappingColumnParseErrorDescription()
+      {
+         Message = Error.InvalidMappingColumn;
+      }
+   }
+
+   public class InconsistentDimensionBetweenUnitsParseErrorDescription : ParseErrorDescription
+   {
+      public InconsistentDimensionBetweenUnitsParseErrorDescription(string mappingName)
+      {
+         Message = Error.InconsistentDimensionBetweenUnitsException(mappingName);
+      }
+   }
+
+   public class ErrorUnitParseErrorDescription : ParseErrorDescription
+   {
+      public ErrorUnitParseErrorDescription()
+      {
+         Message = Error.InvalidErrorDimension;
+      }
+   }
+
+   public class EmptyDataSetsParseErrorDescription : ParseErrorDescription
+   {
+      public EmptyDataSetsParseErrorDescription(IEnumerable<string> dataSetNames)
+      {
+         Message = Error.EmptyDataSet($"{dataSetNames.ToString(", ")}");
+      }
+   }
+}

src/OSPSuite.Infrastructure.Import/Core/Helpers/CachedListHelpers.cs

@@ -0,0 +1,24 @@
+using OSPSuite.Utility.Collections;
+using System.Collections.Generic;
+
+namespace OSPSuite.Infrastructure.Import.Core.Helpers
+{
+   static public class CachedListHelpers
+   {
+      public static void Add<T1, T2>(Cache<T1, List<T2>> cache, T1 key, T2 x)
+      {
+         if (cache.Contains(key))
+            cache[key].Add(x);
+         else
+            cache.Add(key, new List<T2>() { x });
+      }
+
+      public static void Add<T1, T2>(Cache<T1, List<T2>> cache, T1 key, IEnumerable<T2> list)
+      {
+         if (cache.Contains(key))
+            cache[key].AddRange(list);
+         else
+            cache.Add(key, new List<T2>(list));
+      }
+   }
+}