Merge pull request #231 from Trivadis/feature/issue-228-detect-charset

Feature/issue 228 detect charset

Author: PhilippSalvisberg

.gitattributes

@@ -0,0 +1,2 @@
+# Test file encoded in windows-1252
+windows-1252.sql encoding=cp1252

sqlcl/format.js

@@ -23,6 +23,8 @@
 var javaString = Java.type("java.lang.String");
 var javaSystem = Java.type("java.lang.System");
 // java.nio
+var javaByteBuffer = Java.type("java.nio.ByteBuffer");
+var javaCharset = Java.type("java.nio.charset.Charset");
 var javaFiles = Java.type("java.nio.file.Files");
 var javaFileSystems = Java.type("java.nio.file.FileSystems");
 var javaPaths = Java.type("java.nio.file.Paths");
@@ -179,8 +181,8 @@ var readFile = function (file) {
     return new javaString(javaFiles.readAllBytes(file));
 }
 
-var writeFile = function (file, content) {
-    var writer = javaFiles.newBufferedWriter(file);
+var writeFile = function (file, content, charset) {
+    var writer = javaFiles.newBufferedWriter(file, charset);
     writer.write(content);
     writer.close();
 }
@@ -533,36 +535,59 @@ var isMarkdownFile = function (file, markdownExtensions) {
     return false;
 }
 
-var formatMarkdownFile = function (file, formatter, serr) {
-    var original = readFile(file)
-    var p = javaPattern.compile("(```\\s*sql\\s*\\n)(.+?)(\\n```)", javaPattern.DOTALL);
-    var m = p.matcher(original);
-    var result = "";
-    var pos = 0;
-    var consoleOutput = false;
-    if (serr == "all" || serr == "mext") {
-        consoleOutput = true;
-    }
-    var sqlBlock = 0;
-    while (m.find()) {
-        sqlBlock++;
-        ctx.write("#" + sqlBlock + "... ");
-        result += original.substring(pos, m.end(1));
-        if (hasParseErrors(m.group(2), consoleOutput)) {
-            ctx.write("skipped... ")
-            result += original.substring(m.start(2), m.end(3));
-        } else {
-            ctx.write("done... ")
-            result += formatter.format(m.group(2));
-            result += original.substring(m.end(2), m.end(3));
+var detectCharset = function(content) {
+    // rudimentary solution since Apache Tika cannot be used in SQLcl
+    // try default character set of the OS (can be overridden via -Dfile.encoding), then UTF-8, then windows-1252
+    var defaultCharsetName = javaCharset.defaultCharset().name();
+    var charsetNames = [defaultCharsetName, "UTF-8", "windows-1252"];
+    for (var i = 0; i < charsetNames.length; i++) {
+        var cs = javaCharset.forName(charsetNames[i]);
+        try {
+            cs.newDecoder().decode(javaByteBuffer.wrap(content));
+            return cs;
+        } catch(e) {
+            // ignore exception
         }
-        pos = m.end(3);
     }
-    if (original.length > pos) {
-        result += original.substring(pos);
+    return null;
+}
+
+var formatMarkdownFile = function (file, formatter, serr) {
+    var bytes = javaFiles.readAllBytes(file);
+    var charset = detectCharset(bytes);
+    if (charset == null) {
+        ctx.write("skipped due to unknown character set.\n");
+    } else {
+        var original = new javaString(bytes, charset);
+        var p = javaPattern.compile("(```\\s*sql\\s*\\n)(.+?)(\\n```)", javaPattern.DOTALL);
+        var m = p.matcher(original);
+        var result = "";
+        var pos = 0;
+        var consoleOutput = false;
+        if (serr == "all" || serr == "mext") {
+            consoleOutput = true;
+        }
+        var sqlBlock = 0;
+        while (m.find()) {
+            sqlBlock++;
+            ctx.write("#" + sqlBlock + "... ");
+            result += original.substring(pos, m.end(1));
+            if (hasParseErrors(m.group(2), consoleOutput)) {
+                ctx.write("skipped... ")
+                result += original.substring(m.start(2), m.end(3));
+            } else {
+                ctx.write("done... ")
+                result += formatter.format(m.group(2));
+                result += original.substring(m.end(2), m.end(3));
+            }
+            pos = m.end(3);
+        }
+        if (original.length > pos) {
+            result += original.substring(pos);
+        }
+        writeFile(file, result, charset);
+        ctx.write("done.\n");
     }
-    writeFile(file, result);
-    ctx.write("done.\n");
 }
 
 var getLineSeparator = function (input) {
@@ -578,16 +603,22 @@ var getLineSeparator = function (input) {
 }
 
 var formatFile = function (file, formatter, serr) {
-    var original = readFile(file)
-    var consoleOutput = false;
-    if (serr == "all" || serr == "ext") {
-        consoleOutput = true;
-    }
-    if (hasParseErrors(original, consoleOutput)) {
-        ctx.write("skipped.\n");
+    var bytes = javaFiles.readAllBytes(file);
+    var charset = detectCharset(bytes);
+    if (charset == null) {
+         ctx.write("skipped due to unknown character set.\n");
     } else {
-        writeFile(file, formatter.format(original) + getLineSeparator(original));
-        ctx.write("done.\n");
+        var original = new javaString(bytes, charset);
+        var consoleOutput = false;
+        if (serr == "all" || serr == "ext") {
+            consoleOutput = true;
+        }
+        if (hasParseErrors(original, consoleOutput)) {
+            ctx.write("skipped.\n");
+        } else {
+            writeFile(file, formatter.format(original) + getLineSeparator(original), charset);
+            ctx.write("done.\n");
+        }
     }
 }
 

standalone/README.md

@@ -51,7 +51,7 @@ The parameters are the same as for the [executable JAR](#executable-jar).
 
 ## How to Build
 
-1. [Download](https://www.oracle.com/tools/downloads/sqlcl-downloads.html) and install SQLcl 22.2.0
+1. [Download](https://www.oracle.com/tools/downloads/sqlcl-downloads.html) and install SQLcl 22.2.1
 2. [Download](https://github.com/graalvm/graalvm-ce-builds/releases) and install the GraalVM JDK 17 22.2.0
 3. Go to the `bin` directory of the GraalVM JDK and run `./gu install js native-image`, if you want to produce a native image
 4. [Download](https://maven.apache.org/download.cgi) and install Apache Maven 3.8.6

standalone/pom.xml

@@ -386,6 +386,10 @@
                                     <classpath>
                                         ${project.build.directory}/${project.build.finalName}.jar:${settings.localRepository}/oracle/dbtools/dbtools-common/${sqlcl.version}/dbtools-common-${sqlcl.version}.jar:${settings.localRepository}/org/reflections/reflections/${reflections.version}/reflections-${reflections.version}.jar:${settings.localRepository}/org/javassist/javassist/${javassist.version}/javassist-${javassist.version}.jar:${settings.localRepository}/org/slf4j/slf4j-api/${slf4j.version}/slf4j-api-${slf4j.version}.jar:${settings.localRepository}/org/slf4j/slf4j-jdk14/${slf4j.version}/slf4j-jdk14-${slf4j.version}.jar:${settings.localRepository}/org/graalvm/js/js-scriptengine/${graalvm.version}/js-scriptengine-${graalvm.version}.jar
                                     </classpath>
+                                    <buildArgs combine.children="append">
+                                        <!-- including all charsets works for build-native only!?! -->
+                                        <buildArg>-H:+AddAllCharsets</buildArg>
+                                    </buildArgs>
                                 </configuration>
                             </execution>
                         </executions>

standalone/src/main/resources/META-INF/native-image/com.trivadis.plsql.formatter/tvdformat/reflect-config.json

@@ -124,6 +124,28 @@
     "allPublicFields": true,
     "allPublicClasses": true
   },
+  {
+    "name": "java.nio.ByteBuffer",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "java.nio.charset.Charset",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
   {
     "name": "java.nio.file.Files",
     "allDeclaredConstructors": true,
@@ -190,6 +212,17 @@
     "allPublicFields": true,
     "allPublicClasses": true
   },
+  {
+    "name": "java.util.Collections$UnmodifiableSortedMap",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
   {
     "name": "java.util.HashMap",
     "allDeclaredConstructors": true,
@@ -267,6 +300,17 @@
     "allPublicFields": true,
     "allPublicClasses": true
   },
+  {
+    "name": "java.util.SortedMap",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
   {
     "name": "java.util.stream.Collectors",
     "allDeclaredConstructors": true,
@@ -542,5 +586,38 @@
   },
   {
     "name": "org.graalvm.polyglot.management.Management"
+  },
+  {
+    "name": "sun.nio.cs.MS1252$Holder",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "sun.nio.cs.SingleByte$Decoder",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
+  },
+  {
+    "name": "sun.nio.cs.UTF_8$Decoder",
+    "allDeclaredConstructors": true,
+    "allDeclaredMethods": true,
+    "allDeclaredFields": true,
+    "allDeclaredClasses": true,
+    "allPublicConstructors": true,
+    "allPublicMethods": true,
+    "allPublicFields": true,
+    "allPublicClasses": true
   }
-]
+]

standalone/src/test/java/com/trivadis/plsql/formatter/sqlcl/tests/AbstractSqlclTest.java

@@ -10,7 +10,6 @@
 import java.nio.file.Paths;
 import java.util.function.Predicate;
 import java.util.logging.LogManager;
-import java.util.stream.Collectors;
 
 public abstract class AbstractSqlclTest {
     static {
@@ -49,6 +48,7 @@ public void reset() {
         setup();
     }
 
+    @SuppressWarnings("resource")
     @BeforeEach
     public void setup() {
         byteArrayOutputStream.reset();
@@ -68,8 +68,7 @@ public void setup() {
             var url = Thread.currentThread().getContextClassLoader().getResource("unformatted");
             assert url != null;
             var unformattedDir = Paths.get(url.getPath());
-            var sources = Files.walk(unformattedDir).filter(Files::isRegularFile)
-                    .collect(Collectors.toList());
+            var sources = Files.walk(unformattedDir).filter(Files::isRegularFile).toList();
             for (Path source : sources) {
                 Path target = Paths.get(tempDir.toString() + File.separator + source.getFileName());
                 Files.copy(source, target);