TIKA-1997 Problem in Tika().detect for xml file signed in CADES

tika-core/src/main/java/org/apache/tika/detect/BerValue.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.util.ArrayList;
+
+class BerValue {
+    static BerValue[] next(byte[] input) {
+        return next(input, 0, input.length);
+    }
+
+    static BerValue[] next(byte[] input, int from, int to) {
+        final ArrayList<BerValue> list = new ArrayList<>();
+        BerValue e;
+        while ((e = next(input, from, to, true)) != null) {
+            list.add(e);
+            // increase the offset by current total length
+            from += e.header + e.length + (e.isIndefinite() ? 2 : 0);
+        }
+        // if there are trailing bytes the input is not a valid DER
+        if (from == to) {
+            // convert the list to array
+            return list.toArray(new BerValue[list.size()]);
+        } else {
+            return null;
+        }
+    }
+
+    static BerValue next(byte[] input, int from, int to, boolean recurse) {
+        if (to > from + 1 && input.length >= to) {
+            final int type = input[from] & 0xff;
+            final int tagClass = type >> 6;
+            final int tag = type & 0x1f;
+            // standard tags only
+            if (tag != 31) {
+                int length = input[from + 1] & 0xff;
+                int header = 2;
+                if ((length & 0x80) != 0) {
+                    final int bytes = length & 0x7f;
+                    if (bytes == 0) {
+                        // indefinite form
+                        if (recurse) {
+                            length = 0;
+                            int off = from + header;
+                            BerValue bv;
+                            while (!((bv = next(input, off + length, to, recurse)) == null || bv.isEOC())) {
+                                length += bv.header + bv.length + (bv.isIndefinite() ? 2 : 0);
+                            }
+                            if (bv != null) {
+                                return new BerValue(tagClass, tag, header, from, length, true);
+                            }
+                        } else {
+                            return new BerValue(tagClass, tag, header, from, -1, true);
+                        }
+                    } else if (bytes < 4) {
+                        // definite long form.
+                        // accept 4 bytes (integer) length at most.
+                        header += bytes;
+                        if ((from + header) <= to) {
+                            length = input[from + 2] & 0xff;
+                            for (int i = 3; i < header; i++) {
+                                length = length << 8 | (input[from + i] & 0xff);
+                            }
+                            return new BerValue(tagClass, tag, header, from, length, false);
+                        }
+                    }
+                } else {
+                    // definite short form
+                    return new BerValue(tagClass, tag, header, from, length, false);
+                }
+            }
+        }
+        return null;
+    }
+
+    private int tagClass;
+    private int tag;
+    private int header;
+    private int offset;
+    private int length;
+    private boolean indef;
+
+    private BerValue(int tagClass, int tag, int header, int offset, int length, boolean indef) {
+        this.tagClass = tagClass;
+        this.tag = tag;
+        this.header = header;
+        this.offset = offset;
+        this.length = length;
+        this.indef = indef;
+    }
+
+    public int getHeader() {
+        return header;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+    public int getOffset() {
+        return offset;
+    }
+
+    boolean isContext(int tag) {
+        return tagClass == 2 && this.tag == tag;
+    }
+
+    boolean isEOC() {
+        return tagClass == 0 && tag == 0;
+    }
+
+    public boolean isIndefinite() {
+        return indef;
+    }
+
+    boolean isInteger() {
+        return tagClass == 0 && tag == 2;
+    }
+
+    boolean isOID() {
+        return tagClass == 0 && tag == 6;
+    }
+
+    boolean isSequence() {
+        return tagClass == 0 && tag == 16;
+    }
+
+    boolean isSet() {
+        return tagClass == 0 && tag == 17;
+    }
+}

tika-core/src/main/java/org/apache/tika/detect/Pkcs7Detector.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Base64.Decoder;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+
+public class Pkcs7Detector implements Detector {
+    private static final long serialVersionUID = 4651588855075311797L;
+
+    public static MediaType PKCS7_MIME = MediaType.application("pkcs7-mime");
+    public static MediaType PKCS7_SIGNATURE = MediaType.application("pkcs7-signature");
+    private static MediaType COMPRESSED_DATA = new MediaType(PKCS7_MIME, "smime-type", "compressed-data");
+    private static MediaType ENVELOPED_DATA = new MediaType(PKCS7_MIME, "smime-type", "enveloped-data");
+    private static MediaType CERTS_ONLY = new MediaType(PKCS7_MIME, "smime-type", "certs-only");
+    private static MediaType SIGNED_DATA = new MediaType(PKCS7_MIME, "smime-type", "signed-data");
+
+    private static byte[] PKCS = { 0x2a, (byte) 0x86, 0x48, (byte) 0x86, (byte) 0xf7, 0x0d, 0x01 };
+    private static char[] BOUNDARY = "-----BEGIN PKCS7-----".toCharArray();
+
+    private byte[] decodePem(byte[] buf, int from, int to) {
+        try {
+            ByteArrayInputStream is = new ByteArrayInputStream(buf);
+            BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.US_ASCII));
+            // consume the encapsulation boundary
+            reader.readLine();
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            Decoder decoder = Base64.getDecoder();
+            String s;
+            while ((s = reader.readLine()) != null && s.charAt(0) != '-') {
+                if ((s.length() & 0x03) != 0) {
+                    // round to the closest multiple of 4
+                    os.write(decoder.decode(s.substring(0, ((s.length() >> 2) << 2)).getBytes(StandardCharsets.US_ASCII)));
+                } else {
+                    os.write(decoder.decode(s.getBytes(StandardCharsets.US_ASCII)));
+                }
+            }
+            return os.toByteArray();
+        } catch (Exception e) {
+            return buf;
+        }
+    }
+
+    private MediaType detect(byte[] input, int from, int to, Metadata metadata) {
+        if (isPem(input, from, to)) {
+            input = decodePem(input, from, to);
+            from = 0;
+            to = input.length;
+        }
+        BerValue ber = BerValue.next(input, from, to, false);
+        if (ber != null && ber.isSequence()) {
+            from += ber.getHeader();
+            ber = BerValue.next(input, from, to, false);
+            if (ber != null && ber.isOID()) {
+                from += ber.getHeader();
+                if (isPKCS(input, from, from + PKCS.length)) {
+                    from += PKCS.length;
+                    int left = ber.getLength() - PKCS.length;
+                    if (isCompressedData(input, from, to, left)) {
+                        return COMPRESSED_DATA;
+                    } else if (isEnvelopedData(input, from, to, left)) {
+                        return ENVELOPED_DATA;
+                    } else if (isSignedData(input, from, to, left)) {
+                        from += left;
+                        // read all "digestAlgorithms"
+                        ber = detectDigestAlgorithms(input, from, to);
+                        if (ber != null) {
+                            if (ber.getLength() == 0) {
+                                // no digest algorithm means no data was signed
+                                return CERTS_ONLY;
+                            }
+                            from = ber.getOffset() + ber.getHeader() + ber.getLength() + (ber.isIndefinite() ? 2 : 0);
+                            ber = BerValue.next(input, from, to, false);
+                            if (ber != null && ber.isSequence()) {
+                                from += ber.getHeader();
+                                if (!ber.isIndefinite()) {
+                                    // limit to "encapContentInfo" not to
+                                    // mistake "certificates" for "eContent"
+                                    // (both are context[0])
+                                    to = Math.min(to, from + ber.getLength());
+                                }
+                                ber = BerValue.next(input, from, to, false);
+                                if (ber != null && ber.isOID()) {
+                                    from += ber.getHeader() + ber.getLength();
+                                    ber = BerValue.next(input, from, to, false);
+                                    if (ber == null || ber.isEOC()) {
+                                        return PKCS7_SIGNATURE;
+                                    } else {
+                                        return SIGNED_DATA;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return MediaType.OCTET_STREAM;
+    }
+
+    public MediaType detect(byte[] input, Metadata metadata) {
+        if (input == null) {
+            return MediaType.OCTET_STREAM;
+        }
+        return detect(input, 0, input.length, metadata);
+    }
+
+    @Override
+    public MediaType detect(InputStream input, Metadata metadata) throws IOException {
+        if (input == null) {
+            return MediaType.OCTET_STREAM;
+        }
+        byte[] buffer = new byte[512];
+        int offset = 0, length;
+        while ((length = input.read(buffer, offset, buffer.length - offset)) != -1 && offset < buffer.length) {
+            offset += length;
+        }
+        return detect(buffer, 0, offset, metadata);
+    }
+
+    private BerValue detectDigestAlgorithms(byte[] input, int from, int to) {
+        BerValue ber = BerValue.next(input, from, to, false);
+        if (ber != null && ber.isContext(0)) {
+            from += ber.getHeader();
+            ber = BerValue.next(input, from, to, false);
+            if (ber != null && ber.isSequence()) {
+                from += ber.getHeader();
+                ber = BerValue.next(input, from, to, false);
+                if (ber != null && ber.isInteger()) {
+                    from += ber.getHeader() + ber.getLength();
+                    ber = BerValue.next(input, from, to, true);
+                    if (ber != null && ber.isSet()) {
+                        return ber;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    private boolean isCompressedData(byte[] input, int from, int to, int left) {
+        if (left == 4 && input.length > from + 3 && input[from] == 0x09 && input[from + 1] == 0x10
+                && input[from + 2] == 0x01 && input[from + 3] == 0x09) {
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isEnvelopedData(byte[] input, int from, int to, int left) {
+        if (left == 2 && input.length > from + 1 && input[from] == 0x07 && input[from + 1] == 0x03) {
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isPem(byte[] buf, int from, int to) {
+        if (to >= from + BOUNDARY.length && buf.length >= to) {
+            for (int i = 0; i < BOUNDARY.length; i++) {
+                if (buf[from + i] != BOUNDARY[i]) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isPKCS(byte[] input, int from, int to) {
+        if (input.length < to) {
+            return false;
+        }
+        for (int i = from; i < to; i++) {
+            if (input[i] != PKCS[i - from]) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean isSignedData(byte[] input, int from, int to, int left) {
+        if (left == 2 && input.length > from + 1 && input[from] == 0x07 && input[from + 1] == 0x02) {
+            return true;
+        }
+        return false;
+    }
+}

tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java

@@ -36,6 +36,7 @@
 
 import org.apache.tika.Tika;
 import org.apache.tika.detect.Detector;
+import org.apache.tika.detect.Pkcs7Detector;
 import org.apache.tika.detect.TextDetector;
 import org.apache.tika.detect.XmlRootExtractor;
 import org.apache.tika.metadata.Metadata;
@@ -241,6 +242,11 @@ List<MimeType> getMimeType(byte[] data) {
                             result.set(i, textMimeType);
                         }
                     }
+                } else if (Pkcs7Detector.PKCS7_MIME.equals(matched.getType())) {
+                    MediaType type = new Pkcs7Detector().detect(data, new Metadata());
+                    if (!MediaType.OCTET_STREAM.equals(type)) {
+                        result.set(i, new MimeType(type));
+                    }
                 }
             }
             return result;