8348014: Enhance certificate processing

Reviewed-by: andrew
Backport-of: b470d96dd18f380044611e2343f08c4db5b495b4

Author: Aleksei Voitylov

src/java.base/share/classes/sun/security/provider/X509Factory.java

@@ -66,6 +66,7 @@ public class X509Factory extends CertificateFactorySpi {
     public static final String END_CERT = "-----END CERTIFICATE-----";
 
     private static final int ENC_MAX_LENGTH = 4096 * 1024; // 4 MB MAX
+    public static final int BER_ITERATION_COUNT = 128; // Limit nested depth
 
     private static final Cache<Object, X509CertImpl> certCache
         = Cache.newSoftMemoryCache(750);
@@ -555,7 +556,7 @@ private static byte[] readOneBlock(InputStream is) throws IOException {
         if (c == DerValue.tag_Sequence) {
             ByteArrayOutputStream bout = new ByteArrayOutputStream(2048);
             bout.write(c);
-            readBERInternal(is, bout, c);
+            readBERInternal(is, bout, c, BER_ITERATION_COUNT);
             return bout.toByteArray();
         } else {
             try {
@@ -579,12 +580,16 @@ private static byte[] readOneBlock(InputStream is) throws IOException {
      * @param is    Read from this InputStream
      * @param bout  Write into this OutputStream
      * @param tag   Tag already read (-1 mean not read)
+     * @param depth nesting depth limit
      * @return     The current tag, used to check EOC in indefinite-length BER
      * @throws IOException Any parsing error
      */
     private static int readBERInternal(InputStream is,
-            ByteArrayOutputStream bout, int tag) throws IOException {
+        ByteArrayOutputStream bout, int tag, int depth) throws IOException {
 
+        if (depth-- == 0) {
+            throw new IOException("Nesting sequence depth limit reached.");
+        }
         if (tag == -1) {        // Not read before the call, read now
             tag = is.read();
             if (tag == -1) {
@@ -610,7 +615,7 @@ private static int readBERInternal(InputStream is,
                         "Non constructed encoding must have definite length");
             }
             while (true) {
-                int subTag = readBERInternal(is, bout, -1);
+                int subTag = readBERInternal(is, bout, -1, depth);
                 if (subTag == 0) {   // EOC, end of indefinite-length section
                     break;
                 }