8000 Unify file encoding when a cmdlet creates a file by JamesWTruher · Pull Request #4119 · PowerShell/PowerShell · GitHub
[go: up one dir, main page]
Skip to content

Unify file encoding when a cmdlet creates a file #4119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 14 commits into from
Closed

Unify file encoding when a cmdlet creates a file #4119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
965d325
Unify file encoding and set new default to UTF8 without BOM
JamesWTruher Jun 19, 2017
afbf374
Fix test issue where newline was being provided rather than calculated
JamesWTruher Jun 26, 2017
f996e6f
Remove code which was relying on previous encoding code
JamesWTruher Jun 26, 2017
a067de4
fix another couple of xplat issues
JamesWTruher Jun 26, 2017
9b8e4fb
Return static Utf8NoBom encoder rather than creating a new instance
JamesWTruher Jun 29, 2017
5309b90
Change expected encoding to not distinguish based on platform except …
JamesWTruher Jun 29, 2017
9a45f40
Remove commented out code
JamesWTruher Jul 11, 2017
65c255c
refactor tests to use fewer instances of hardcoded byte strings
JamesWTruher Jul 19, 2017
14933a9
update tests to include tests against Encoding probe method for exist…
JamesWTruher Jul 20, 2017
acac547
Refactor Encoding.cs to improve readability
JamesWTruher Jul 21, 2017 8000
40a8eee
Change FileEncoding.Unknown to FileEncoding.Unspecified which more cl…
JamesWTruher Jul 25, 2017
bf2be6f
Changed PowerShellEncoding class name to EncodingUtils
JamesWTruher Jul 26, 2017
8a0765e
update PowerShellEncoding to use new name EncodingUtils
JamesWTruher Jul 26, 2017
ff24e22
Use new class name EncodingUtils
JamesWTruher Jul 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.IO;
using System.Linq;
using System.Management.Automation;
using Microsoft.PowerShell;
using System.Text;
using Dbg = System.Management.Automation.Diagnostics;

Expand Down Expand Up @@ -206,8 +207,7 @@ public SwitchParameter NoClobber
/// Encoding optional flag
/// </summary>
[Parameter()]
[ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
public string Encoding { get; set; }
public FileEncoding Encoding { get; set; } = FileEncoding.Unspecified;

/// <summary>
/// Property that sets append parameter.
Expand Down Expand Up @@ -367,7 +367,7 @@ private void CreateFileStream()
PathUtils.MasterStreamOpen(
this,
this.Path,
Encoding ?? "ASCII",
Encoding,
false, // defaultEncoding
Append,
Force,
Expand Down Expand Up @@ -571,8 +571,7 @@ public SwitchParameter UseCulture
/// Encoding optional flag
/// </summary>
[Parameter()]
[ValidateSetAttribute(new[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
public string Encoding { get; set; }
public FileEncoding Encoding { get; set; } = FileEncoding.Unspecified;

/// <summary>
/// Avoid writing out duplicate warning messages when there are
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.PowerShell.Commands.Utility/commands/utility/ExportAliasCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ private StreamWriter OpenFile(out FileInfo readOnlyFileInfo)
PathUtils.MasterStreamOpen(
this,
this.Path,
EncodingConversion.Unicode,
FileEncoding.Unicode,
false, // defaultEncoding
Append,
Force,
Expand Down
11 changes: 2 additions & 9 deletions ...oft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,7 @@ public sealed class FormatHex : PSCmdlet
/// Type of character encoding for InputObject
/// </summary>
[Parameter(ParameterSetName = "ByInputObject")]
[ValidateSetAttribute(new string[] {
EncodingConversion.Unicode,
EncodingConversion.BigEndianUnicode,
EncodingConversion.Utf8,
EncodingConversion.Utf7,
EncodingConversion.Utf32,
EncodingConversion.Ascii})]
public string Encoding { get; set; } = "Ascii";
public FileEncoding Encoding { get; set; } = FileEncoding.UTF8NoBOM;

/// <summary>
/// This parameter is no-op
Expand Down Expand Up @@ -239,7 +232,7 @@ private void ProcessObjectContent(PSObject inputObject)
else if (obj is string)
{
string inputString = obj.ToString();
Encoding resolvedEncoding = EncodingConversion.Convert(this, Encoding);
Encoding resolvedEncoding = EncodingUtils.GetEncoding(this, Encoding);
inputBytes = resolvedEncoding.GetBytes(inputString);
}

Expand Down
17 changes: 3 additions & 14 deletions ...crosoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Management.Automation.Host;
using System.IO;
using Microsoft.PowerShell.Commands.Internal.Format;
using Microsoft.PowerShell;

namespace Microsoft.PowerShell.Commands
{
Expand Down Expand Up @@ -72,25 +73,13 @@ public string LiteralPath
/// </summary>
///
[Parameter(Position = 1)]
[ValidateNotNullOrEmpty]
[ValidateSetAttribute(new string[] {
EncodingConversion.Unknown,
EncodingConversion.String,
EncodingConversion.Unicode,
EncodingConversion.BigEndianUnicode,
EncodingConversion.Utf8,
EncodingConversion.Utf7,
EncodingConversion.Utf32,
EncodingConversion.Ascii,
EncodingConversion.Default,
EncodingConversion.OEM })]
public string Encoding
public FileEncoding Encoding
{
get { return _encoding; }
set { _encoding = value; }
}

private string _encoding;
private FileEncoding _encoding;

/// <summary>
/// Property that sets append parameter.
Expand Down
16 changes: 2 additions & 14 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,19 +77,7 @@ public SwitchParameter Force
/// Encoding optional flag
/// </summary>
[Parameter]
[ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
public string Encoding
{
get
{
return _encoding.GetType().Name;
}
set
{
_encoding = EncodingConversion.Convert(this, value);
}
}
private Encoding _encoding = System.Text.Encoding.UTF8;
public FileEncoding Encoding { get; set; } = FileEncoding.Unspecified;

#endregion Parameters

Expand Down Expand Up @@ -144,7 +132,7 @@ protected override void BeginProcessing()
List<string> generatedFiles = GenerateProxyModule(
tempDirectory,
Path.GetFileName(directory.FullName),
_encoding,
EncodingUtils.GetEncoding(this, Encoding),
_force,
listOfCommandMetadata,
alias2resolvedCommandName,
Expand Down
16 changes: 3 additions & 13 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1200,17 +1200,7 @@ public SwitchParameter AllMatches
/// The text encoding to process each file as.
/// </summary>
[Parameter]
[ValidateNotNullOrEmpty]
[ValidateSetAttribute(new string[] {
EncodingConversion.Unicode,
EncodingConversion.Utf7,
EncodingConversion.Utf8,
EncodingConversion.Utf32,
EncodingConversion.Ascii,
EncodingConversion.BigEndianUnicode,
EncodingConversion.Default,
EncodingConversion.OEM })]
public string Encoding { get; set; }
public FileEncoding Encoding { get; set; } = FileEncoding.Unspecified;

private System.Text.Encoding _textEncoding;

Expand Down Expand Up @@ -1282,9 +1272,9 @@ public SwitchParameter AllMatches
protected override void BeginProcessing()
{
// Process encoding switch.
if (Encoding != null)
if (Encoding != FileEncoding.Unspecified )
{
_textEncoding = EncodingConversion.Convert(this, Encoding);
_textEncoding = EncodingUtils.GetEncoding(this, Encoding);
}
else
{
Expand Down
30 changes: 9 additions & 21 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Net.Mail;
using System.Diagnostics.CodeAnalysis;
using System.Management.Automation;
using Microsoft.PowerShell;


namespace Microsoft.PowerShell.Commands
Expand Down Expand Up @@ -492,35 +493,22 @@ protected override void EndProcessing()

/// <summary>
/// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here.
/// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do
/// the conversion and return the converted value. Otherwise, we just return the input data.
/// When the input data is of type string and is valid to be converted to System.Text.Encoding
/// via EncodingUtils.GetEncoding(), we do the conversion and return the converted value.
/// Otherwise, we just return the input data.
/// </summary>
internal sealed class ArgumentToEncodingNameTransformationAttribute : ArgumentTransformationAttribute
{
public override object Transform(EngineIntrinsics engineIntrinsics, object inputData)
{
string encodingName;
if (LanguagePrimitives.TryConvertTo<string>(inputData, out encodingName))
{
if (string.Equals(encodingName, EncodingConversion.Unknown, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.String, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Unicode, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.BigEndianUnicode, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Utf8, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Utf7, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Utf32, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Ascii, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.Default, StringComparison.OrdinalIgnoreCase) ||
string.Equals(encodingName, EncodingConversion.OEM, StringComparison.OrdinalIgnoreCase))
{
// the encodingName is guaranteed to be valid, so it is safe to pass null to method
// Convert(Cmdlet cmdlet, string encoding) as the value of 'cmdlet'.
return EncodingConversion.Convert(null, encodingName);
}
FileEncoding encoding;
if (LanguagePrimitives.TryConvertTo<FileEncoding>(inputData, out encoding))
{
return EncodingUtils.GetEncoding(encoding);
}
return inputData;
}
}

#endregion
}
}
4 changes: 2 additions & 2 deletions src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using System.Security;
using System.Text;
using System.Xml;
using Microsoft.PowerShell;
using Dbg = System.Management.Automation.Diagnostics;

namespace Microsoft.PowerShell.Commands
Expand Down Expand Up @@ -108,8 +109,7 @@ public SwitchParameter NoClobber
/// </summary>
///
[Parameter]
[ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
public string Encoding { get; set; } = "Unicode";
public FileEncoding Encoding { get; set; } = FileEncoding.Unspecified;

#endregion Command Line Parameters

Expand Down
3 changes: 2 additions & 1 deletion src/System.Management.Automation/commands/utility/FormatAndOutput/common/FormatXMLWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Collections.Generic;
using System.Globalization;
using System.Xml;
using Microsoft.PowerShell;

namespace Microsoft.PowerShell.Commands
{
Expand Down Expand Up @@ -37,7 +38,7 @@ internal static void WriteToPs1Xml(PSCmdlet cmdlet, List<ExtendedTypeDefinition>
StreamWriter streamWriter;
FileStream fileStream;
FileInfo fileInfo;
PathUtils.MasterStreamOpen(cmdlet, filepath, "ascii", true, false, force, noclobber,
PathUtils.MasterStreamOpen(cmdlet, filepath, FileEncoding.Ascii, true, false, force, noclobber,
out fileStream, out streamWriter, out fileInfo, isLiteralPath);

try
Expand Down
3 changes: 2 additions & 1 deletion src/System.Management.Automation/engine/ExternalScriptInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Text;
using System.Collections.ObjectModel;
using System.Management.Automation.Runspaces;
using Microsoft.PowerShell;
using Microsoft.PowerShell.Commands;
using System.Collections.Generic;
using System.Management.Automation.Language;
Expand Down Expand Up @@ -467,7 +468,7 @@ private void ReadScriptContents()
{
using (FileStream readerStream = new FileStream(_path, FileMode.Open, FileAccess.Read))
{
Encoding defaultEncoding = ClrFacade.GetDefaultEncoding();
Encoding defaultEncoding = EncodingUtils.GetDefaultEncoding();
Microsoft.Win32.SafeHandles.SafeFileHandle safeFileHandle = readerStream.SafeFileHandle;

using (StreamReader scriptReader = new StreamReader(readerStream, defaultEncoding))
Expand Down
11 changes: 10 additions & 1 deletion src/System.Management.Automation/engine/InitialSessionState.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/********************************************************************++
/*********************************************************************++
Copyright (c) Microsoft Corporation. All rights reserved.
--********************************************************************/

Expand All @@ -15,6 +15,7 @@
using System.Management.Automation.Language;
using System.Reflection;
using System.Threading;
using Microsoft.PowerShell;
using Microsoft.PowerShell.Commands;
using Debug = System.Management.Automation.Diagnostics;
using System.Management.Automation.Host;
Expand Down Expand Up @@ -4828,6 +4829,7 @@ .ForwardHelpCategory Cmdlet
internal const ActionPreference defaultVerbosePreference = ActionPreference.SilentlyContinue;
internal const ActionPreference defaultWarningPreference = ActionPreference.Continue;
internal const ActionPreference defaultInformationPreference = ActionPreference.SilentlyContinue;
internal const Microsoft.PowerShell.FileEncoding defaultFileEncodingPreference = FileEncoding.Unspecified;
internal const bool defaultWhatIfPreference = false;
internal const ConfirmImpact defaultConfirmPreference = ConfirmImpact.High;

Expand Down Expand Up @@ -4908,6 +4910,13 @@ .ForwardHelpCategory Cmdlet
ScopedItemOptions.None,
new ArgumentTypeConverterAttribute(typeof(ActionPreference))
),
new SessionStateVariableEntry(
SpecialVariables.DefaultFileEncodingPreference,
defaultFileEncodingPreference,
RunspaceInit.DefaultFileEncodingDescription,
ScopedItemOptions.None,
new ArgumentTypeConverterAttribute(typeof(Microsoft.PowerShell.FileEncoding))
),
new SessionStateVariableEntry(
SpecialVariables.ErrorView,
"NormalView",
Expand Down
9 changes: 3 additions & 6 deletions src/System.Management.Automation/engine/Modules/NewModuleManifestCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using System.Management.Automation;
using System.Management.Automation.Internal;
using System.Diagnostics.CodeAnalysis;
using Microsoft.PowerShell;
using Dbg = System.Management.Automation.Diagnostics;

//
Expand Down Expand Up @@ -940,12 +941,8 @@ protected override void EndProcessing()
// Now open the output file...
PathUtils.MasterStreamOpen(
this,
filePath,
#if UNIX
new UTF8Encoding(false), // UTF-8, no BOM
#else
EncodingConversion.Unicode, // UTF-16 with BOM
#endif
filePath,
FileEncoding.Unspecified,
/* defaultEncoding */ false,
/* Append */ false,
/* Force */ false,
Expand Down
5 changes: 3 additions & 2 deletions src/System.Management.Automation/engine/Modules/ScriptAnalysis.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Text;
using System.Management.Automation.Language;
using System.Text.RegularExpressions;
using Microsoft.PowerShell;

namespace System.Management.Automation
{
Expand Down Expand Up @@ -95,7 +96,7 @@ internal static string ReadScript(string path)
{
using (FileStream readerStream = new FileStream(path, FileMode.Open, FileAccess.Read))
{
Encoding defaultEncoding = ClrFacade.GetDefaultEncoding();
Encoding defaultEncoding = EncodingUtils.GetDefaultEncoding();
Microsoft.Win32.SafeHandles.SafeFileHandle safeFileHandle = readerStream.SafeFileHandle;

using (StreamReader scriptReader = new StreamReader(readerStream, defaultEncoding))
Expand Down Expand Up @@ -554,4 +555,4 @@ internal class RequiredModuleInfo
internal string Name { get; set; }
internal List<string> CommandsToPostFilter { get; set; }
}
} // System.Management.Automation
} // System.Management.Automation
3 changes: 3 additions & 0 deletions src/System.Management.Automation/engine/SpecialVariables.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,9 @@ internal static class SpecialVariables
internal const string InformationPreference = "InformationPreference";
internal static readonly VariablePath InformationPreferenceVarPath = new VariablePath(InformationPreference);

internal const string DefaultFileEncodingPreference = "PSDefaultFileEncoding";
internal static readonly VariablePath DefaultFileEncodingVarPath = new VariablePath(DefaultFileEncodingPreference);

#endregion Preference Variables

internal const string ErrorView = "ErrorView";
Expand Down
Loading
0