Add formatter for the charset .editorconfig option

src/CodeFormatter.cs

@@ -24,6 +24,7 @@ internal static class CodeFormatter
             new WhitespaceFormatter(),
             new FinalNewlineFormatter(),
             new EndOfLineFormatter(),
+            new CharsetFormatter(),
         }.ToImmutableArray();
 
         public static async Task<WorkspaceFormatResult> FormatWorkspaceAsync(

src/Formatters/CharsetFormatter.cs

@@ -0,0 +1,71 @@
+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.CodingConventions;
+
+namespace Microsoft.CodeAnalysis.Tools.Formatters
+{
+    internal sealed class CharsetFormatter : DocumentFormatter
+    {
+        protected override string FormatWarningDescription => Resources.Fix_file_encoding;
+
+        private static Encoding Utf8 => new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
+        private static Encoding Latin1 => Encoding.GetEncoding("iso-8859-1");
+
+        protected override Task<SourceText> FormatFileAsync(
+            Document document,
+            SourceText sourceText,
+            OptionSet options,
+            ICodingConventionsSnapshot codingConventions,
+            FormatOptions formatOptions,
+            ILogger logger,
+            CancellationToken cancellationToken)
+        {
+            return Task.Run(() =>
+            {
+                if (!TryGetCharset(codingConventions, out var encoding)
+                    || sourceText.Encoding.Equals(encoding))
+                {
+                    return sourceText;
+                }
+
+                return SourceText.From(sourceText.ToString(), encoding);
+            });
+        }
+
+        private static bool TryGetCharset(ICodingConventionsSnapshot codingConventions, out Encoding encoding)
+        {
+            if (codingConventions.TryGetConventionValue("charset", out string charsetOption))
+            {
+                encoding = GetCharset(charsetOption);
+                return true;
+            }
+
+            encoding = null;
+            return false;
+        }
+
+        public static Encoding GetCharset(string charsetOption)
+        {
+            switch (charsetOption)
+            {
+                case "latin1":
+                    return Latin1;
+                case "utf-8-bom":
+                    return Encoding.UTF8; // UTF-8 with BOM Marker
+                case "utf-16be":
+                    return Encoding.BigEndianUnicode; // Big Endian with BOM Marker
+                case "utf-16le":
+                    return Encoding.Unicode; // Little Endian with BOM Marker
+                case "utf-8":
+                default:
+                    return Utf8;
+            }
+        }
+    }
+}

src/Formatters/DocumentFormatter.cs

@@ -81,7 +81,7 @@ protected abstract Task<SourceText> FormatFileAsync(
             var originalSourceText = await document.GetTextAsync(cancellationToken).ConfigureAwait(false);
             var formattedSourceText = await FormatFileAsync(document, originalSourceText, options, codingConventions, formatOptions, logger, cancellationToken).ConfigureAwait(false);
 
-            return !formattedSourceText.ContentEquals(originalSourceText)
+            return !formattedSourceText.ContentEquals(originalSourceText) || !formattedSourceText.Encoding.Equals(originalSourceText.Encoding)
                 ? (originalSourceText, formattedSourceText)
                 : (originalSourceText, null);
         }
@@ -117,7 +117,7 @@ private async Task<Solution> ApplyFileChangesAsync(
                     LogFormattingChanges(formatOptions.WorkspaceFilePath, document.FilePath, originalText, formattedText, formatOptions.ChangesAreErrors, logger);
                 }
 
-                formattedSolution = formattedSolution.WithDocumentText(document.Id, formattedText);
+                formattedSolution = formattedSolution.WithDocumentText(document.Id, formattedText, PreservationMode.PreserveIdentity);
             }
 
             return formattedSolution;

src/Resources.resx

@@ -201,4 +201,7 @@
   <data name="Fix_whitespace_formatting" xml:space="preserve">
     <value>Fix whitespace formatting.</value>
   </data>
+  <data name="Fix_file_encoding" xml:space="preserve">
+    <value>Fix file encoding.</value>
+  </data>
 </root>

src/xlf/Resources.cs.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.de.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.es.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.fr.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.it.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.ja.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.ko.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.pl.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.pt-BR.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.ru.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.tr.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.zh-Hans.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

src/xlf/Resources.zh-Hant.xlf

@@ -47,6 +47,11 @@
         <target state="new">Fix end of line marker.</target>
         <note />
       </trans-unit>
+      <trans-unit id="Fix_file_encoding">
+        <source>Fix file encoding.</source>
+        <target state="new">Fix file encoding.</target>
+        <note />
+      </trans-unit>
       <trans-unit id="Fix_whitespace_formatting">
         <source>Fix whitespace formatting.</source>
         <target state="new">Fix whitespace formatting.</target>

tests/Formatters/AbstractFormatterTests.cs

@@ -5,6 +5,7 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.Host.Mef;
@@ -66,50 +67,67 @@ protected AbstractFormatterTest()
         /// </value>
         public abstract string Language { get; }
 
-        private string TestCode
-        {
-            set
-            {
-                if (value != null)
-                {
-                    TestState.Sources.Add(value);
-                }
-            }
-        }
-
         private static ILogger Logger => new TestLogger();
         private static EditorConfigOptionsApplier OptionsApplier => new EditorConfigOptionsApplier();
 
         public SolutionState TestState { get; }
 
-        private protected async Task TestAsync(string testCode, string expectedCode, IReadOnlyDictionary<string, string> editorConfig)
+        private protected Task<SourceText> TestAsync(string testCode, string expectedCode, IReadOnlyDictionary<string, string> editorConfig)
+        {
+            return TestAsync(testCode, expectedCode, editorConfig, Encoding.UTF8);
+        }
+
+        private protected async Task<SourceText> TestAsync(string testCode, string expectedCode, IReadOnlyDictionary<string, string> editorConfig, Encoding encoding)
         {
-            TestCode = testCode;
+            var text = SourceText.From(testCode, encoding);
+            TestState.Sources.Add(text);
 
             var solution = GetSolution(TestState.Sources.ToArray(), TestState.AdditionalFiles.ToArray(), TestState.AdditionalReferences.ToArray());
             var project = solution.Projects.Single();
             var document = project.Documents.Single();
-            var options = (OptionSet)await document.GetOptionsAsync();
             var formatOptions = new FormatOptions(
                 workspaceFilePath: project.FilePath,
                 isSolution: false,
                 logLevel: LogLevel.Trace,
                 saveFormattedFiles: false,
                 changesAreErrors: false,
-                filesToFormat: ImmutableHashSet.Create<string>(document.FilePath));
-
-            ICodingConventionsSnapshot codingConventions = new TestCodingConventionsSnapshot(editorConfig);
-            options = OptionsApplier.ApplyConventions(options, codingConventions, Language);
+                filesToFormat: ImmutableHashSet.Create(document.FilePath));
 
-            var filesToFormat = new[] { (document, options, codingConventions) }.ToImmutableArray();
+            var filesToFormat = await GetOnlyFileToFormatAsync(solution, editorConfig);
 
             var formattedSolution = await Formatter.FormatAsync(solution, filesToFormat, formatOptions, Logger, default);
-            var formattedDocument = formattedSolution.Projects.Single().Documents.Single();
+            var formattedDocument = GetOnlyDocument(formattedSolution);
             var formattedText = await formattedDocument.GetTextAsync();
 
             Assert.Equal(expectedCode, formattedText.ToString());
+
+            return formattedText;
         }
 
+        /// <summary>
+        /// Gets the only <see cref="Document"/> along with related options and conventions.
+        /// </summary>
+        /// <param name="solution">A Solution containing a single Project containing a single Document.</param>
+        /// <param name="editorConfig">The editorconfig to apply to the documents options set.</param>
+        /// <returns>The document contained within along with option set and coding conventions.</returns>
+        protected async Task<ImmutableArray<(Document, OptionSet, ICodingConventionsSnapshot)>> GetOnlyFileToFormatAsync(Solution solution, IReadOnlyDictionary<string, string> editorConfig)
+        {
+            var document = GetOnlyDocument(solution);
+            var options = (OptionSet)await document.GetOptionsAsync();
+
+            ICodingConventionsSnapshot codingConventions = new TestCodingConventionsSnapshot(editorConfig);
+            options = OptionsApplier.ApplyConventions(options, codingConventions, Language);
+
+            return ImmutableArray.Create((document, options, codingConventions));
+        }
+
+        /// <summary>
+        /// Gets the only <see cref="Document"/> contained within the only <see cref="Project"/> within the <see cref="Solution"/>.
+        /// </summary>
+        /// <param name="solution">A Solution containing a single Project containing a single Document.</param>
+        /// <returns>The document contained within.</returns>
+        public Document GetOnlyDocument(Solution solution) => solution.Projects.Single().Documents.Single();
+
         /// <summary>
         /// Gets the collection of inputs to provide to the XML documentation resolver.
         /// </summary>
@@ -125,13 +143,6 @@ private protected async Task TestAsync(string testCode, string expectedCode, IRe
         /// </summary>
         public List<Func<OptionSet, OptionSet>> OptionsTransforms { get; } = new List<Func<OptionSet, OptionSet>>();
 
-        public Document GetTestDocument(string testCode)
-        {
-            TestCode = testCode;
-            var solution = GetSolution(TestState.Sources.ToArray(), TestState.AdditionalFiles.ToArray(), TestState.AdditionalReferences.ToArray());
-            return solution.Projects.Single().Documents.Single();
-        }
-
         /// <summary>
         /// Given an array of strings as sources and a language, turn them into a <see cref="Project"/> and return the
         /// solution.