Add --no-bom flag for ODBC sqlcmd compatibility

By default, -u (unicode output) includes a UTF-16 LE BOM (FF FE) at the
start of output files. ODBC sqlcmd does not write a BOM.

This adds --no-bom flag to omit the BOM when strict ODBC compatibility
is needed. Usage: sqlcmd -u --no-bom -o output.txt

Changes:
- Add NoBOM field to SQLCmdArguments and Sqlcmd structs
- Add --no-bom flag with descriptive help
- Conditionally use unicode.IgnoreBOM when flag is set
- Update README to document the difference and new flag
- Add TestUnicodeOutputNoBOM test

Author: dlevy-msft-sql

README.md

@@ -132,7 +132,7 @@ The following switches have different behavior in this version of `sqlcmd` compa
   - If both `-N` and `-C` are provided, sqlcmd will use their values for encryption negotiation.
   - To provide the value of the host name in the server certificate when using strict encryption, pass the host name with `-F`. Example: `-Ns -F myhost.domain.com`
   - More information about client/server encryption negotiation can be found at <https://docs.microsoft.com/openspecs/windows_protocols/ms-tds/60f56408-0188-4cd5-8b90-25c6f2423868>
-- `-u` The generated Unicode output file will have the UTF16 Little-Endian Byte-order mark (BOM) written to it.
+- `-u` The generated Unicode output file will have the UTF16 Little-Endian Byte-order mark (BOM) written to it. ODBC sqlcmd does not write a BOM; use `--no-bom` with `-u` if you need strict ODBC compatibility.
 - Some behaviors that were kept to maintain compatibility with `OSQL` may be changed, such as alignment of column headers for some data types.
 - All commands must fit on one line, even `EXIT`. Interactive mode will not check for open parentheses or quotes for commands and prompt for successive lines. The ODBC sqlcmd allows the query run by `EXIT(query)` to span multiple lines.
 - `-i` doesn't handle a comma `,` in a file name correctly unless the file name argument is triple quoted. For example:

cmd/sqlcmd/sqlcmd.go

@@ -66,6 +66,7 @@ type SQLCmdArguments struct {
 	ErrorsToStderr              *int
 	Headers                     int
 	UnicodeOutputFile           bool
+	NoBOM                       bool
 	Version                     bool
 	ColumnSeparator             string
 	ScreenWidth                 *int
@@ -457,6 +458,7 @@ func setFlags(rootCmd *cobra.Command, args *SQLCmdArguments) {
 	rootCmd.Flags().IntVarP(&args.Headers, "headers", "h", 0, localizer.Sprintf("Specifies the number of rows to print between the column headings. Use -h-1 to specify that headers not be printed"))
 
 	rootCmd.Flags().BoolVarP(&args.UnicodeOutputFile, "unicode-output-file", "u", false, localizer.Sprintf("Specifies that all output files are encoded with little-endian Unicode"))
+	rootCmd.Flags().BoolVar(&args.NoBOM, "no-bom", false, localizer.Sprintf("Omit the UTF-16 BOM from Unicode output files. Use with -u for ODBC sqlcmd compatibility"))
 	rootCmd.Flags().StringVarP(&args.ColumnSeparator, "column-separator", "s", "", localizer.Sprintf("Specifies the column separator character. Sets the %s variable.", localizer.ColSeparatorVar))
 	rootCmd.Flags().BoolVarP(&args.TrimSpaces, "trim-spaces", "W", false, localizer.Sprintf("Remove trailing spaces from a column"))
 	_ = rootCmd.Flags().BoolP("multi-subnet-failover", "M", false, localizer.Sprintf("Provided for backward compatibility. Sqlcmd always optimizes detection of the active replica of a SQL Failover Cluster"))
@@ -812,6 +814,7 @@ func run(vars *sqlcmd.Variables, args *SQLCmdArguments) (int, error) {
 	s.SetupCloseHandler()
 	defer s.StopCloseHandler()
 	s.UnicodeOutputFile = args.UnicodeOutputFile
+	s.NoBOM = args.NoBOM
 
 	if args.DisableCmd != nil {
 		s.Cmd.DisableSysCommands(args.errorOnBlockedCmd())

cmd/sqlcmd/sqlcmd_test.go

@@ -298,6 +298,32 @@ func TestUnicodeOutput(t *testing.T) {
 	}
 }
 
+func TestUnicodeOutputNoBOM(t *testing.T) {
+	o, err := os.CreateTemp("", "sqlcmdnobom")
+	assert.NoError(t, err, "os.CreateTemp")
+	defer os.Remove(o.Name())
+	defer o.Close()
+	args = newArguments()
+	args.InputFile = []string{"testdata/selectutf8.txt"}
+	args.OutputFile = o.Name()
+	args.UnicodeOutputFile = true
+	args.NoBOM = true
+	setAzureAuthArgIfNeeded(&args)
+	vars := sqlcmd.InitializeVariables(args.useEnvVars())
+	setVars(vars, &args)
+
+	exitCode, err := run(vars, &args)
+	assert.NoError(t, err, "run")
+	assert.Equal(t, 0, exitCode, "exitCode")
+	fileBytes, err := os.ReadFile(o.Name())
+	if assert.NoError(t, err, "os.ReadFile") {
+		// With --no-bom, the file should NOT start with FF FE (UTF-16 LE BOM)
+		assert.True(t, len(fileBytes) >= 2, "output file should have content")
+		hasBOM := len(fileBytes) >= 2 && fileBytes[0] == 0xFF && fileBytes[1] == 0xFE
+		assert.False(t, hasBOM, "output file should NOT have BOM when --no-bom is used")
+	}
+}
+
 func TestUnicodeInput(t *testing.T) {
 	testfiles := []string{
 		filepath.Join(`testdata`, `selectutf8.txt`),

pkg/sqlcmd/commands.go

@@ -321,9 +321,12 @@ func outCommand(s *Sqlcmd, args []string, line uint) error {
 			return InvalidFileError(err, args[0])
 		}
 		if s.UnicodeOutputFile {
-			// ODBC sqlcmd doesn't write a BOM but we will.
-			// Maybe the endian-ness should be configurable.
-			win16le := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM)
+			// By default we write a BOM, but --no-bom omits it for ODBC sqlcmd compatibility
+			bomPolicy := unicode.UseBOM
+			if s.NoBOM {
+				bomPolicy = unicode.IgnoreBOM
+			}
+			win16le := unicode.UTF16(unicode.LittleEndian, bomPolicy)
 			encoder := transform.NewWriter(o, win16le.NewEncoder())
 			s.SetOutput(encoder)
 		} else {

pkg/sqlcmd/sqlcmd.go

@@ -84,6 +84,8 @@ type Sqlcmd struct {
 	PrintError func(msg string, severity uint8) bool
 	// UnicodeOutputFile is true when UTF16 file output is needed
 	UnicodeOutputFile bool
+	// NoBOM omits the BOM from UTF-16 output files (ODBC sqlcmd compatibility)
+	NoBOM bool
 	// EchoInput tells the GO command to print the batch text before running the query
 	EchoInput bool
 	colorizer color.Colorizer