diff --git a/src/DocGenerator/DocGenerator.csproj b/src/DocGenerator/DocGenerator.csproj index ea937de..b1e744d 100644 --- a/src/DocGenerator/DocGenerator.csproj +++ b/src/DocGenerator/DocGenerator.csproj @@ -33,13 +33,11 @@ 4 - - ..\packages\CommandLineParser.2.1.1-beta\lib\net45\CommandLine.dll - True + + ..\packages\CommandLineParser.2.3.0\lib\net45\CommandLine.dll - - ..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll - True + + ..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll diff --git a/src/DocGenerator/packages.config b/src/DocGenerator/packages.config index f71c972..80bd0f3 100644 --- a/src/DocGenerator/packages.config +++ b/src/DocGenerator/packages.config @@ -1,5 +1,5 @@  - - + + \ No newline at end of file diff --git a/src/XrmCommandBox.IntegrationTests/App.config b/src/XrmCommandBox.IntegrationTests/App.config index ba5df3f..5bb0e5f 100644 --- a/src/XrmCommandBox.IntegrationTests/App.config +++ b/src/XrmCommandBox.IntegrationTests/App.config @@ -21,4 +21,20 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/XrmCommandBox.IntegrationTests/XrmCommandBox.IntegrationTests.csproj b/src/XrmCommandBox.IntegrationTests/XrmCommandBox.IntegrationTests.csproj index 1dc73b0..b686f41 100644 --- a/src/XrmCommandBox.IntegrationTests/XrmCommandBox.IntegrationTests.csproj +++ b/src/XrmCommandBox.IntegrationTests/XrmCommandBox.IntegrationTests.csproj @@ -36,19 +36,37 @@ 4 - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0.2\lib\net452\Microsoft.Crm.Sdk.Proxy.dll + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Crm.Sdk.Proxy.dll - - ..\packages\Microsoft.IdentityModel.6.1.7600.16394\lib\net35\Microsoft.IdentityModel.dll + + ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0.2\lib\net452\Microsoft.Xrm.Sdk.dll + + ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.WindowsForms.dll + + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Rest.ClientRuntime.dll + + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.dll + + + ..\packages\Microsoft.CrmSdk.Deployment.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Deployment.dll + + + ..\packages\Microsoft.CrmSdk.Workflow.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Workflow.dll + + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Xrm.Tooling.Connector.dll - ..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll + ..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll + + + @@ -57,6 +75,10 @@ + + + + diff --git a/src/XrmCommandBox.IntegrationTests/packages.config b/src/XrmCommandBox.IntegrationTests/packages.config index 0c19605..43bb525 100644 --- a/src/XrmCommandBox.IntegrationTests/packages.config +++ b/src/XrmCommandBox.IntegrationTests/packages.config @@ -1,6 +1,9 @@  - - - + + + + + + \ No newline at end of file diff --git a/src/XrmCommandBox.Tests/XrmCommandBox.Tests.csproj b/src/XrmCommandBox.Tests/XrmCommandBox.Tests.csproj index e5e116e..7badc3e 100644 --- a/src/XrmCommandBox.Tests/XrmCommandBox.Tests.csproj +++ b/src/XrmCommandBox.Tests/XrmCommandBox.Tests.csproj @@ -38,47 +38,40 @@ ..\packages\FakeItEasy.3.2.0\lib\net40\FakeItEasy.dll - True - - ..\packages\FakeXrmEasy.365.1.36.1\lib\net452\FakeXrmEasy.dll - True + + ..\packages\FakeXrmEasy.9.1.47.0\lib\net452\FakeXrmEasy.dll ..\packages\log4net.2.0.8\lib\net45-full\log4net.dll True - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0.2\lib\net452\Microsoft.Crm.Sdk.Proxy.dll - True - - - ..\packages\Microsoft.IdentityModel.6.1.7600.16394\lib\net35\Microsoft.IdentityModel.dll - True + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Crm.Sdk.Proxy.dll ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - True ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.WindowsForms.dll - True - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0.2\lib\net452\Microsoft.Xrm.Sdk.dll - True + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Rest.ClientRuntime.dll - - ..\packages\Microsoft.CrmSdk.Deployment.8.2.0.2\lib\net452\Microsoft.Xrm.Sdk.Deployment.dll - True + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.dll - - ..\packages\Microsoft.CrmSdk.Workflow.8.2.0.2\lib\net452\Microsoft.Xrm.Sdk.Workflow.dll - True + + ..\packages\Microsoft.CrmSdk.Deployment.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Deployment.dll - - ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.8.2.0.4\lib\net452\Microsoft.Xrm.Tooling.Connector.dll - True + + ..\packages\Microsoft.CrmSdk.Workflow.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Workflow.dll + + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Xrm.Tooling.Connector.dll + + + ..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll diff --git a/src/XrmCommandBox.Tests/app.config b/src/XrmCommandBox.Tests/app.config index 144df92..bdf94a6 100644 --- a/src/XrmCommandBox.Tests/app.config +++ b/src/XrmCommandBox.Tests/app.config @@ -15,4 +15,13 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/XrmCommandBox.Tests/packages.config b/src/XrmCommandBox.Tests/packages.config index 5503cb9..dfbf3dd 100644 --- a/src/XrmCommandBox.Tests/packages.config +++ b/src/XrmCommandBox.Tests/packages.config @@ -1,13 +1,12 @@  - - + - - - - - + + + + + \ No newline at end of file diff --git a/src/XrmCommandBox/App.config b/src/XrmCommandBox/App.config index 0dcfa97..318221a 100644 --- a/src/XrmCommandBox/App.config +++ b/src/XrmCommandBox/App.config @@ -78,4 +78,20 @@ --> + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/XrmCommandBox/Data/DatasetSerializer.cs b/src/XrmCommandBox/Data/DatasetSerializer.cs new file mode 100644 index 0000000..b0e30df --- /dev/null +++ b/src/XrmCommandBox/Data/DatasetSerializer.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace XrmCommandBox.Data +{ + class DataSetSerializer + { + public DataSet Deserialize(string file) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/XrmCommandBox/Tools/Common/Utilities.cs b/src/XrmCommandBox/Tools/Common/Utilities.cs new file mode 100644 index 0000000..68f4b1c --- /dev/null +++ b/src/XrmCommandBox/Tools/Common/Utilities.cs @@ -0,0 +1,78 @@ +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace XrmCommandBox.Tools.Common +{ + public static class Utilities + { + internal static Guid? GetExistingRecordId(IOrganizationService service, string entityName, Entity entityRecord, IList matchAttributes, EntityMetadata entityMetadata) + { + Guid? recordGuid = null; + + var qry = GetMatchQuery(entityName, entityRecord, matchAttributes, entityMetadata); + + var foundRecords = service.RetrieveMultiple(qry); + + if (foundRecords.Entities.Count > 0) + { + if (foundRecords.Entities.Count > 1) + throw new Exception("Too many records found"); + + recordGuid = foundRecords.Entities[0].Id; + } + + return recordGuid; + } + + private static QueryBase GetMatchQuery(string entityName, Entity entityRecord, IList matchAttributes, EntityMetadata entityMetadata) + { + if (matchAttributes == null || matchAttributes.Count == 0) + { + // set the id attribute as match attribute + var attrId = entityMetadata.PrimaryIdAttribute; + + matchAttributes = new[] { attrId }; + } + + var qry = new QueryByAttribute + { + EntityName = entityName, + ColumnSet = new ColumnSet(entityMetadata.PrimaryIdAttribute) + }; + + qry.Attributes.AddRange(matchAttributes); + + foreach (var attrName in matchAttributes) + { + var filterAttrValue = entityRecord.Contains(attrName) ? GetFilterValue(entityRecord[attrName]) : null; + qry.Values.Add(filterAttrValue); + } + + return qry; + } + + private static object GetFilterValue(object attributeValue) + { + var filterValue = attributeValue; + + if (attributeValue is EntityReference) + { + var attrValueReference = (EntityReference)attributeValue; + filterValue = attrValueReference.Id; + } + else if (attributeValue is OptionSetValue) + { + var attrValueOptionset = (OptionSetValue)attributeValue; + filterValue = attrValueOptionset.Value; + } + + return filterValue; + } + } +} diff --git a/src/XrmCommandBox/Tools/DataLoader/ColumnMappingOptions.cs b/src/XrmCommandBox/Tools/DataLoader/ColumnMappingOptions.cs new file mode 100644 index 0000000..91f2ca4 --- /dev/null +++ b/src/XrmCommandBox/Tools/DataLoader/ColumnMappingOptions.cs @@ -0,0 +1,6 @@ +namespace XrmCommandBox.Tools.DataLoader +{ + public class ColumnMappingOptions + { + } +} \ No newline at end of file diff --git a/src/XrmCommandBox/Tools/DataLoader/DataLoaderTool.cs b/src/XrmCommandBox/Tools/DataLoader/DataLoaderTool.cs new file mode 100644 index 0000000..687a274 --- /dev/null +++ b/src/XrmCommandBox/Tools/DataLoader/DataLoaderTool.cs @@ -0,0 +1,119 @@ +using log4net; +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; +using Microsoft.Xrm.Sdk.Query; +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; +using XrmCommandBox.Data; +using XrmCommandBox.Tools.Common; + +namespace XrmCommandBox.Tools.DataLoader +{ + public class DataLoaderTool + { + private readonly IOrganizationService _crmService; + private readonly ILog _log = LogManager.GetLogger(typeof(DataLoaderTool)); + + public DataLoaderTool(IOrganizationService service) + { + _crmService = service; + } + + public void Run(DataLoaderToolOptions options) + { + var sw = Stopwatch.StartNew(); + int recordCount = 0, createdCount = 0, updatedCount = 0, errorsCount = 0, progress = 0; + var serializer = new DataSetSerializer(); + + _log.Info("Running DataLoader Tool..."); + if (options.MappingOptions == null || options.MappingOptions.Count() == 0) throw new Exception("Mappings not defined"); + + _log.Info($"Reading {options.File} file..."); + + var dataset = serializer.Deserialize(options.File); + + if (dataset == null) throw new Exception("Unexpected error reading file"); + + _log.Info($"{dataset.Tables.Count} tables read"); + + foreach (System.Data.DataTable dataTable in dataset.Tables) + { + int dtCreatedCount = 0, dtUpdatedCount = 0, dtErrorsCount = 0; + ProcessDataSet(dataTable, options, out dtCreatedCount, out dtUpdatedCount, out dtErrorsCount); + } + + sw.Stop(); + _log.Info($"Done! Processed {recordCount} records in {sw.Elapsed.TotalSeconds.ToString("0.00")} seconds. Created: {createdCount}. Updated: {updatedCount}. Errors: {errorsCount}"); + } + + + private void ProcessDataSet(System.Data.DataTable dataTable, DataLoaderToolOptions options, out int createdCount, out int updatedCount, out int errorCount) + { + int recordCount = 0; + int progress = 0; + + createdCount = 0; + updatedCount = 0; + errorCount = 0; + + // find the options for this datatable + var mappingOptions = options.MappingOptions.FirstOrDefault(x => string.Compare(x.TableName, dataTable.TableName, true) == 0); + + if (mappingOptions != null) + { + _log.Debug($"Querying metadata of entity {mappingOptions.EntityName}..."); + var metadata = _crmService.GetMetadata(mappingOptions.EntityName); + + _log.Info("Processing records..."); + var records = dataTable.AsEntityCollection(metadata, mappingOptions); + + foreach (var entityRecord in records.Entities) + { + try + { + recordCount++; + progress = (int)Math.Round(((decimal)recordCount / records.Entities.Count) * 100); // calculate the progress percentage + _log.Info($"{entityRecord.LogicalName} {recordCount} of {records.Entities.Count} : {entityRecord.Id} ({progress}%)"); + + // figure out if the record exists, in order to decide to create or update it + var recordId = Utilities.GetExistingRecordId(_crmService, entityRecord.LogicalName, entityRecord, mappingOptions.MatchAttributes?.ToList(), metadata); + _log.Debug($"RecordId: {recordId}"); + + if (recordId != null) + { + // the record exists, so update it + _log.Info($"Updating record: {recordId}..."); + entityRecord[metadata.PrimaryIdAttribute] = recordId.Value; + entityRecord.Id = recordId.Value; + _crmService.Update(entityRecord); + _log.Info("Record updated successfully"); + updatedCount++; + } + else + { + // the record doesn't exist, so create it + _log.Info("Creating record...."); + recordId = _crmService.Create(entityRecord); + _log.Info($"Record created successfully: Guid {recordId}"); + createdCount++; + } + } + catch (Exception ex) + { + errorCount++; + _log.Error(ex); + if (!options.ContinueOnError) throw; + } + } + } + else + { + _log.Info($"No mappings found for table {dataTable.TableName}"); + } + } + + } +} \ No newline at end of file diff --git a/src/XrmCommandBox/Tools/DataLoader/DataLoaderToolOptions.cs b/src/XrmCommandBox/Tools/DataLoader/DataLoaderToolOptions.cs new file mode 100644 index 0000000..f67163a --- /dev/null +++ b/src/XrmCommandBox/Tools/DataLoader/DataLoaderToolOptions.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using System.Xml.Serialization; +using CommandLine; + +namespace XrmCommandBox.Tools.DataLoader +{ + [Verb("import", HelpText = "Imports data to Dynamics")] + [Handler(typeof(DataLoaderTool))] + public class DataLoaderToolOptions : CrmCommonOptions + { + [Option('f', "file", HelpText = "File containing the configuration of the data to import")] + public string File { get; set; } + + [Option('e', "continue-on-error", HelpText = "Continue if there's an error while processing the command")] + public bool ContinueOnError { get; set; } + + public IEnumerable MappingOptions; + } +} \ No newline at end of file diff --git a/src/XrmCommandBox/Tools/DataLoader/Extensions.cs b/src/XrmCommandBox/Tools/DataLoader/Extensions.cs new file mode 100644 index 0000000..ce80f11 --- /dev/null +++ b/src/XrmCommandBox/Tools/DataLoader/Extensions.cs @@ -0,0 +1,18 @@ +using Microsoft.Xrm.Sdk; +using Microsoft.Xrm.Sdk.Metadata; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace XrmCommandBox.Tools.DataLoader +{ + public static class Extensions + { + public static EntityCollection AsEntityCollection(this System.Data.DataTable dataTable, EntityMetadata metadata, TableMappingOptions mappingOptions) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/XrmCommandBox/Tools/DataLoader/TableMappingOptions.cs b/src/XrmCommandBox/Tools/DataLoader/TableMappingOptions.cs new file mode 100644 index 0000000..e5d482b --- /dev/null +++ b/src/XrmCommandBox/Tools/DataLoader/TableMappingOptions.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; + +namespace XrmCommandBox.Tools.DataLoader +{ + public class TableMappingOptions + { + public string TableName { get; internal set; } + public string EntityName { get; internal set; } + + public IEnumerable ColumnMappings; + + public IEnumerable MatchAttributes { get; set; } + } +} \ No newline at end of file diff --git a/src/XrmCommandBox/XrmCommandBox.csproj b/src/XrmCommandBox/XrmCommandBox.csproj index 17b461d..5ced28e 100644 --- a/src/XrmCommandBox/XrmCommandBox.csproj +++ b/src/XrmCommandBox/XrmCommandBox.csproj @@ -43,6 +43,7 @@ + @@ -50,6 +51,12 @@ + + + + + + @@ -80,14 +87,15 @@ Designer - + + Designer + - - ..\packages\CommandLineParser.2.1.1-beta\lib\net45\CommandLine.dll - True + + ..\packages\CommandLineParser.2.3.0\lib\net45\CommandLine.dll ..\packages\ExcelDataReader.3.4.1\lib\net45\ExcelDataReader.dll @@ -96,51 +104,50 @@ ..\packages\log4net.2.0.8\lib\net45-full\log4net.dll True - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0\lib\net45\Microsoft.Crm.Sdk.Proxy.dll - True - - - ..\packages\Microsoft.IdentityModel.6.1.7600.16394\lib\net35\Microsoft.IdentityModel.dll - True + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Crm.Sdk.Proxy.dll ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - True ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.2.22.302111727\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.WindowsForms.dll - True - - ..\packages\Microsoft.CrmSdk.CoreAssemblies.8.2.0\lib\net45\Microsoft.Xrm.Sdk.dll - True + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Rest.ClientRuntime.dll - - ..\packages\Microsoft.CrmSdk.Deployment.8.2.0\lib\net45\Microsoft.Xrm.Sdk.Deployment.dll - True + + ..\packages\Microsoft.CrmSdk.CoreAssemblies.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.dll - - ..\packages\Microsoft.CrmSdk.Workflow.8.2.0\lib\net45\Microsoft.Xrm.Sdk.Workflow.dll - True + + ..\packages\Microsoft.CrmSdk.Deployment.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Deployment.dll - - ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.8.2.0.5\lib\net452\Microsoft.Xrm.Tooling.Connector.dll - True + + ..\packages\Microsoft.CrmSdk.Workflow.9.0.2.4\lib\net452\Microsoft.Xrm.Sdk.Workflow.dll + + + ..\packages\Microsoft.CrmSdk.XrmTooling.CoreAssembly.9.0.2.5\lib\net452\Microsoft.Xrm.Tooling.Connector.dll - ..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll + ..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll + + + ..\packages\System.Console.4.0.0\lib\net46\System.Console.dll + + + ..\packages\System.Reflection.TypeExtensions.4.1.0\lib\net46\System.Reflection.TypeExtensions.dll + diff --git a/src/XrmCommandBox/packages.config b/src/XrmCommandBox/packages.config index e156a47..0458bad 100644 --- a/src/XrmCommandBox/packages.config +++ b/src/XrmCommandBox/packages.config @@ -1,14 +1,26 @@  - - + + - - - - - + + + + - + + + + + + + + + + + + + + \ No newline at end of file