001/*
002 *  Licensed to the Apache Software Foundation (ASF) under one or more
003 *  contributor license agreements.  See the NOTICE file distributed with
004 *  this work for additional information regarding copyright ownership.
005 *  The ASF licenses this file to You under the Apache License, Version 2.0
006 *  (the "License"); you may not use this file except in compliance with
007 *  the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 *  Unless required by applicable law or agreed to in writing, software
012 *  distributed under the License is distributed on an "AS IS" BASIS,
013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 *  See the License for the specific language governing permissions and
015 *  limitations under the License.
016 *
017 */
018package org.apache.commons.compress.archivers.sevenz;
019
020import java.io.BufferedInputStream;
021import java.io.ByteArrayInputStream;
022import java.io.Closeable;
023import java.io.DataInputStream;
024import java.io.EOFException;
025import java.io.File;
026import java.io.FilterInputStream;
027import java.io.IOException;
028import java.io.InputStream;
029import java.nio.ByteBuffer;
030import java.nio.ByteOrder;
031import java.nio.CharBuffer;
032import java.nio.channels.SeekableByteChannel;
033import java.nio.charset.StandardCharsets;
034import java.nio.charset.CharsetEncoder;
035import java.nio.file.Files;
036import java.nio.file.StandardOpenOption;
037import java.util.ArrayList;
038import java.util.Arrays;
039import java.util.BitSet;
040import java.util.EnumSet;
041import java.util.HashMap;
042import java.util.LinkedList;
043import java.util.List;
044import java.util.Map;
045import java.util.stream.Collectors;
046import java.util.zip.CRC32;
047
048import org.apache.commons.compress.MemoryLimitException;
049import org.apache.commons.compress.utils.BoundedInputStream;
050import org.apache.commons.compress.utils.ByteUtils;
051import org.apache.commons.compress.utils.CRC32VerifyingInputStream;
052import org.apache.commons.compress.utils.IOUtils;
053import org.apache.commons.compress.utils.InputStreamStatistics;
054
055/**
056 * Reads a 7z file, using SeekableByteChannel under
057 * the covers.
058 * <p>
059 * The 7z file format is a flexible container
060 * that can contain many compression and
061 * encryption types, but at the moment only
062 * only Copy, LZMA, LZMA2, BZIP2, Deflate and AES-256 + SHA-256
063 * are supported.
064 * <p>
065 * The format is very Windows/Intel specific,
066 * so it uses little-endian byte order,
067 * doesn't store user/group or permission bits,
068 * and represents times using NTFS timestamps
069 * (100 nanosecond units since 1 January 1601).
070 * Hence the official tools recommend against
071 * using it for backup purposes on *nix, and
072 * recommend .tar.7z or .tar.lzma or .tar.xz
073 * instead.
074 * <p>
075 * Both the header and file contents may be
076 * compressed and/or encrypted. With both
077 * encrypted, neither file names nor file
078 * contents can be read, but the use of
079 * encryption isn't plausibly deniable.
080 *
081 * <p>Multi volume archives can be read by concatenating the parts in
082 * correct order - either manually or by using {link
083 * org.apache.commons.compress.utils.MultiReadOnlySeekableByteChannel}
084 * for example.</p>
085 *
086 * @NotThreadSafe
087 * @since 1.6
088 */
089public class SevenZFile implements Closeable {
090    static final int SIGNATURE_HEADER_SIZE = 32;
091
092    private static final String DEFAULT_FILE_NAME = "unknown archive";
093
094    private final String fileName;
095    private SeekableByteChannel channel;
096    private final Archive archive;
097    private int currentEntryIndex = -1;
098    private int currentFolderIndex = -1;
099    private InputStream currentFolderInputStream;
100    private byte[] password;
101    private final SevenZFileOptions options;
102
103    private long compressedBytesReadFromCurrentEntry;
104    private long uncompressedBytesReadFromCurrentEntry;
105
106    private final ArrayList<InputStream> deferredBlockStreams = new ArrayList<>();
107
108    // shared with SevenZOutputFile and tests, neither mutates it
109    static final byte[] sevenZSignature = { //NOSONAR
110        (byte)'7', (byte)'z', (byte)0xBC, (byte)0xAF, (byte)0x27, (byte)0x1C
111    };
112
113    /**
114     * Reads a file as 7z archive
115     *
116     * @param fileName the file to read
117     * @param password optional password if the archive is encrypted
118     * @throws IOException if reading the archive fails
119     * @since 1.17
120     */
121    public SevenZFile(final File fileName, final char[] password) throws IOException {
122        this(fileName, password, SevenZFileOptions.DEFAULT);
123    }
124
125    /**
126     * Reads a file as 7z archive with additional options.
127     *
128     * @param fileName the file to read
129     * @param password optional password if the archive is encrypted
130     * @param options the options to apply
131     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
132     * @since 1.19
133     */
134    public SevenZFile(final File fileName, final char[] password, final SevenZFileOptions options) throws IOException {
135        this(Files.newByteChannel(fileName.toPath(), EnumSet.of(StandardOpenOption.READ)), // NOSONAR
136                fileName.getAbsolutePath(), utf16Decode(password), true, options);
137    }
138
139    /**
140     * Reads a file as 7z archive
141     *
142     * @param fileName the file to read
143     * @param password optional password if the archive is encrypted -
144     * the byte array is supposed to be the UTF16-LE encoded
145     * representation of the password.
146     * @throws IOException if reading the archive fails
147     * @deprecated use the char[]-arg version for the password instead
148     */
149    @Deprecated
150    public SevenZFile(final File fileName, final byte[] password) throws IOException {
151        this(Files.newByteChannel(fileName.toPath(), EnumSet.of(StandardOpenOption.READ)),
152                fileName.getAbsolutePath(), password, true, SevenZFileOptions.DEFAULT);
153    }
154
155    /**
156     * Reads a SeekableByteChannel as 7z archive
157     *
158     * <p>{@link
159     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
160     * allows you to read from an in-memory archive.</p>
161     *
162     * @param channel the channel to read
163     * @throws IOException if reading the archive fails
164     * @since 1.13
165     */
166    public SevenZFile(final SeekableByteChannel channel) throws IOException {
167        this(channel, SevenZFileOptions.DEFAULT);
168    }
169
170    /**
171     * Reads a SeekableByteChannel as 7z archive with addtional options.
172     *
173     * <p>{@link
174     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
175     * allows you to read from an in-memory archive.</p>
176     *
177     * @param channel the channel to read
178     * @param options the options to apply
179     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
180     * @since 1.19
181     */
182    public SevenZFile(final SeekableByteChannel channel, final SevenZFileOptions options) throws IOException {
183        this(channel, DEFAULT_FILE_NAME, null, options);
184    }
185
186    /**
187     * Reads a SeekableByteChannel as 7z archive
188     *
189     * <p>{@link
190     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
191     * allows you to read from an in-memory archive.</p>
192     *
193     * @param channel the channel to read
194     * @param password optional password if the archive is encrypted
195     * @throws IOException if reading the archive fails
196     * @since 1.17
197     */
198    public SevenZFile(final SeekableByteChannel channel,
199                      final char[] password) throws IOException {
200        this(channel, password, SevenZFileOptions.DEFAULT);
201    }
202
203    /**
204     * Reads a SeekableByteChannel as 7z archive with additional options.
205     *
206     * <p>{@link
207     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
208     * allows you to read from an in-memory archive.</p>
209     *
210     * @param channel the channel to read
211     * @param password optional password if the archive is encrypted
212     * @param options the options to apply
213     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
214     * @since 1.19
215     */
216    public SevenZFile(final SeekableByteChannel channel, final char[] password, final SevenZFileOptions options)
217            throws IOException {
218        this(channel, DEFAULT_FILE_NAME, password, options);
219    }
220
221    /**
222     * Reads a SeekableByteChannel as 7z archive
223     *
224     * <p>{@link
225     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
226     * allows you to read from an in-memory archive.</p>
227     *
228     * @param channel the channel to read
229     * @param fileName name of the archive - only used for error reporting
230     * @param password optional password if the archive is encrypted
231     * @throws IOException if reading the archive fails
232     * @since 1.17
233     */
234    public SevenZFile(final SeekableByteChannel channel, final String fileName,
235                      final char[] password) throws IOException {
236        this(channel, fileName, password, SevenZFileOptions.DEFAULT);
237    }
238
239    /**
240     * Reads a SeekableByteChannel as 7z archive with addtional options.
241     *
242     * <p>{@link
243     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
244     * allows you to read from an in-memory archive.</p>
245     *
246     * @param channel the channel to read
247     * @param fileName name of the archive - only used for error reporting
248     * @param password optional password if the archive is encrypted
249     * @param options the options to apply
250     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
251     * @since 1.19
252     */
253    public SevenZFile(final SeekableByteChannel channel, final String fileName, final char[] password,
254            final SevenZFileOptions options) throws IOException {
255        this(channel, fileName, utf16Decode(password), false, options);
256    }
257
258    /**
259     * Reads a SeekableByteChannel as 7z archive
260     *
261     * <p>{@link
262     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
263     * allows you to read from an in-memory archive.</p>
264     *
265     * @param channel the channel to read
266     * @param fileName name of the archive - only used for error reporting
267     * @throws IOException if reading the archive fails
268     * @since 1.17
269     */
270    public SevenZFile(final SeekableByteChannel channel, final String fileName)
271        throws IOException {
272        this(channel, fileName, SevenZFileOptions.DEFAULT);
273    }
274
275    /**
276     * Reads a SeekableByteChannel as 7z archive with additional options.
277     *
278     * <p>{@link
279     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
280     * allows you to read from an in-memory archive.</p>
281     *
282     * @param channel the channel to read
283     * @param fileName name of the archive - only used for error reporting
284     * @param options the options to apply
285     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
286     * @since 1.19
287     */
288    public SevenZFile(final SeekableByteChannel channel, final String fileName, final SevenZFileOptions options)
289            throws IOException {
290        this(channel, fileName, null, false, options);
291    }
292
293    /**
294     * Reads a SeekableByteChannel as 7z archive
295     *
296     * <p>{@link
297     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
298     * allows you to read from an in-memory archive.</p>
299     *
300     * @param channel the channel to read
301     * @param password optional password if the archive is encrypted -
302     * the byte array is supposed to be the UTF16-LE encoded
303     * representation of the password.
304     * @throws IOException if reading the archive fails
305     * @since 1.13
306     * @deprecated use the char[]-arg version for the password instead
307     */
308    @Deprecated
309    public SevenZFile(final SeekableByteChannel channel,
310                      final byte[] password) throws IOException {
311        this(channel, DEFAULT_FILE_NAME, password);
312    }
313
314    /**
315     * Reads a SeekableByteChannel as 7z archive
316     *
317     * <p>{@link
318     * org.apache.commons.compress.utils.SeekableInMemoryByteChannel}
319     * allows you to read from an in-memory archive.</p>
320     *
321     * @param channel the channel to read
322     * @param fileName name of the archive - only used for error reporting
323     * @param password optional password if the archive is encrypted -
324     * the byte array is supposed to be the UTF16-LE encoded
325     * representation of the password.
326     * @throws IOException if reading the archive fails
327     * @since 1.13
328     * @deprecated use the char[]-arg version for the password instead
329     */
330    @Deprecated
331    public SevenZFile(final SeekableByteChannel channel, final String fileName,
332                      final byte[] password) throws IOException {
333        this(channel, fileName, password, false, SevenZFileOptions.DEFAULT);
334    }
335
336    private SevenZFile(final SeekableByteChannel channel, final String filename,
337                       final byte[] password, final boolean closeOnError, final SevenZFileOptions options) throws IOException {
338        boolean succeeded = false;
339        this.channel = channel;
340        this.fileName = filename;
341        this.options = options;
342        try {
343            archive = readHeaders(password);
344            if (password != null) {
345                this.password = Arrays.copyOf(password, password.length);
346            } else {
347                this.password = null;
348            }
349            succeeded = true;
350        } finally {
351            if (!succeeded && closeOnError) {
352                this.channel.close();
353            }
354        }
355    }
356
357    /**
358     * Reads a file as unencrypted 7z archive
359     *
360     * @param fileName the file to read
361     * @throws IOException if reading the archive fails
362     */
363    public SevenZFile(final File fileName) throws IOException {
364        this(fileName, SevenZFileOptions.DEFAULT);
365    }
366
367    /**
368     * Reads a file as unencrypted 7z archive
369     *
370     * @param fileName the file to read
371     * @param options the options to apply
372     * @throws IOException if reading the archive fails or the memory limit (if set) is too small
373     * @since 1.19
374     */
375    public SevenZFile(final File fileName, final SevenZFileOptions options) throws IOException {
376        this(fileName, null, options);
377    }
378
379    /**
380     * Closes the archive.
381     * @throws IOException if closing the file fails
382     */
383    @Override
384    public void close() throws IOException {
385        if (channel != null) {
386            try {
387                channel.close();
388            } finally {
389                channel = null;
390                if (password != null) {
391                    Arrays.fill(password, (byte) 0);
392                }
393                password = null;
394            }
395        }
396    }
397
398    /**
399     * Returns the next Archive Entry in this archive.
400     *
401     * @return the next entry,
402     *         or {@code null} if there are no more entries
403     * @throws IOException if the next entry could not be read
404     */
405    public SevenZArchiveEntry getNextEntry() throws IOException {
406        if (currentEntryIndex >= archive.files.length - 1) {
407            return null;
408        }
409        ++currentEntryIndex;
410        final SevenZArchiveEntry entry = archive.files[currentEntryIndex];
411        if (entry.getName() == null && options.getUseDefaultNameForUnnamedEntries()) {
412            entry.setName(getDefaultName());
413        }
414        buildDecodingStream(currentEntryIndex, false);
415        uncompressedBytesReadFromCurrentEntry = compressedBytesReadFromCurrentEntry = 0;
416        return entry;
417    }
418
419    /**
420     * Returns a copy of meta-data of all archive entries.
421     *
422     * <p>This method only provides meta-data, the entries can not be
423     * used to read the contents, you still need to process all
424     * entries in order using {@link #getNextEntry} for that.</p>
425     *
426     * <p>The content methods are only available for entries that have
427     * already been reached via {@link #getNextEntry}.</p>
428     *
429     * @return a copy of meta-data of all archive entries.
430     * @since 1.11
431     */
432    public Iterable<SevenZArchiveEntry> getEntries() {
433        return new ArrayList<>(Arrays.asList(archive.files));
434    }
435
436    private Archive readHeaders(final byte[] password) throws IOException {
437        final ByteBuffer buf = ByteBuffer.allocate(12 /* signature + 2 bytes version + 4 bytes CRC */)
438            .order(ByteOrder.LITTLE_ENDIAN);
439        readFully(buf);
440        final byte[] signature = new byte[6];
441        buf.get(signature);
442        if (!Arrays.equals(signature, sevenZSignature)) {
443            throw new IOException("Bad 7z signature");
444        }
445        // 7zFormat.txt has it wrong - it's first major then minor
446        final byte archiveVersionMajor = buf.get();
447        final byte archiveVersionMinor = buf.get();
448        if (archiveVersionMajor != 0) {
449            throw new IOException(String.format("Unsupported 7z version (%d,%d)",
450                    archiveVersionMajor, archiveVersionMinor));
451        }
452
453        boolean headerLooksValid = false;  // See https://www.7-zip.org/recover.html - "There is no correct End Header at the end of archive"
454        final long startHeaderCrc = 0xffffFFFFL & buf.getInt();
455        if (startHeaderCrc == 0) {
456            // This is an indication of a corrupt header - peek the next 20 bytes
457            final long currentPosition = channel.position();
458            final ByteBuffer peekBuf = ByteBuffer.allocate(20);
459            readFully(peekBuf);
460            channel.position(currentPosition);
461            // Header invalid if all data is 0
462            while (peekBuf.hasRemaining()) {
463                if (peekBuf.get()!=0) {
464                    headerLooksValid = true;
465                    break;
466                }
467            }
468        } else {
469            headerLooksValid = true;
470        }
471
472        if (headerLooksValid) {
473            final StartHeader startHeader = readStartHeader(startHeaderCrc);
474            return initializeArchive(startHeader, password, true);
475        }
476        // No valid header found - probably first file of multipart archive was removed too early. Scan for end header.
477        if (options.getTryToRecoverBrokenArchives()) {
478            return tryToLocateEndHeader(password);
479        }
480        throw new IOException("archive seems to be invalid.\nYou may want to retry and enable the"
481            + " tryToRecoverBrokenArchives if the archive could be a multi volume archive that has been closed"
482            + " prematurely.");
483    }
484
485    private Archive tryToLocateEndHeader(final byte[] password) throws IOException {
486        final ByteBuffer nidBuf = ByteBuffer.allocate(1);
487        final long searchLimit = 1024L * 1024 * 1;
488        // Main header, plus bytes that readStartHeader would read
489        final long previousDataSize = channel.position() + 20;
490        final long minPos;
491        // Determine minimal position - can't start before current position
492        if (channel.position() + searchLimit > channel.size()) {
493            minPos = channel.position();
494        } else {
495            minPos = channel.size() - searchLimit;
496        }
497        long pos = channel.size() - 1;
498        // Loop: Try from end of archive
499        while (pos > minPos) {
500            pos--;
501            channel.position(pos);
502            nidBuf.rewind();
503            if (channel.read(nidBuf) < 1) {
504                throw new EOFException();
505            }
506            final int nid = nidBuf.array()[0];
507            // First indicator: Byte equals one of these header identifiers
508            if (nid == NID.kEncodedHeader || nid == NID.kHeader) {
509                try {
510                    // Try to initialize Archive structure from here
511                    final StartHeader startHeader = new StartHeader();
512                    startHeader.nextHeaderOffset = pos - previousDataSize;
513                    startHeader.nextHeaderSize = channel.size() - pos;
514                    final Archive result = initializeArchive(startHeader, password, false);
515                    // Sanity check: There must be some data...
516                    if (result.packSizes.length > 0 && result.files.length > 0) {
517                        return result;
518                    }
519                } catch (final Exception ignore) {
520                    // Wrong guess...
521                }
522            }
523        }
524        throw new IOException("Start header corrupt and unable to guess end header");
525    }
526
527    private Archive initializeArchive(final StartHeader startHeader, final byte[] password, final boolean verifyCrc) throws IOException {
528        assertFitsIntoNonNegativeInt("nextHeaderSize", startHeader.nextHeaderSize);
529        final int nextHeaderSizeInt = (int) startHeader.nextHeaderSize;
530        channel.position(SIGNATURE_HEADER_SIZE + startHeader.nextHeaderOffset);
531        ByteBuffer buf = ByteBuffer.allocate(nextHeaderSizeInt).order(ByteOrder.LITTLE_ENDIAN);
532        readFully(buf);
533        if (verifyCrc) {
534            final CRC32 crc = new CRC32();
535            crc.update(buf.array());
536            if (startHeader.nextHeaderCrc != crc.getValue()) {
537                throw new IOException("NextHeader CRC mismatch");
538            }
539        }
540
541        Archive archive = new Archive();
542        int nid = getUnsignedByte(buf);
543        if (nid == NID.kEncodedHeader) {
544            buf = readEncodedHeader(buf, archive, password);
545            // Archive gets rebuilt with the new header
546            archive = new Archive();
547            nid = getUnsignedByte(buf);
548        }
549        if (nid != NID.kHeader) {
550            throw new IOException("Broken or unsupported archive: no Header");
551        }
552        readHeader(buf, archive);
553        archive.subStreamsInfo = null;
554        return archive;
555    }
556
557    private StartHeader readStartHeader(final long startHeaderCrc) throws IOException {
558        final StartHeader startHeader = new StartHeader();
559        // using Stream rather than ByteBuffer for the benefit of the
560        // built-in CRC check
561        try (DataInputStream dataInputStream = new DataInputStream(new CRC32VerifyingInputStream(
562                new BoundedSeekableByteChannelInputStream(channel, 20), 20, startHeaderCrc))) {
563             startHeader.nextHeaderOffset = Long.reverseBytes(dataInputStream.readLong());
564             if (startHeader.nextHeaderOffset < 0
565                 || startHeader.nextHeaderOffset + SIGNATURE_HEADER_SIZE > channel.size()) {
566                 throw new IOException("nextHeaderOffset is out of bounds");
567             }
568
569             startHeader.nextHeaderSize = Long.reverseBytes(dataInputStream.readLong());
570             final long nextHeaderEnd = startHeader.nextHeaderOffset + startHeader.nextHeaderSize;
571             if (nextHeaderEnd < startHeader.nextHeaderOffset
572                 || nextHeaderEnd + SIGNATURE_HEADER_SIZE > channel.size()) {
573                 throw new IOException("nextHeaderSize is out of bounds");
574             }
575
576             startHeader.nextHeaderCrc = 0xffffFFFFL & Integer.reverseBytes(dataInputStream.readInt());
577
578             return startHeader;
579        }
580    }
581
582    private void readHeader(final ByteBuffer header, final Archive archive) throws IOException {
583        final int pos = header.position();
584        final ArchiveStatistics stats = sanityCheckAndCollectStatistics(header);
585        stats.assertValidity(options.getMaxMemoryLimitInKb());
586        header.position(pos);
587
588        int nid = getUnsignedByte(header);
589
590        if (nid == NID.kArchiveProperties) {
591            readArchiveProperties(header);
592            nid = getUnsignedByte(header);
593        }
594
595        if (nid == NID.kAdditionalStreamsInfo) {
596            throw new IOException("Additional streams unsupported");
597            //nid = getUnsignedByte(header);
598        }
599
600        if (nid == NID.kMainStreamsInfo) {
601            readStreamsInfo(header, archive);
602            nid = getUnsignedByte(header);
603        }
604
605        if (nid == NID.kFilesInfo) {
606            readFilesInfo(header, archive);
607            nid = getUnsignedByte(header);
608        }
609    }
610
611    private ArchiveStatistics sanityCheckAndCollectStatistics(final ByteBuffer header)
612        throws IOException {
613        final ArchiveStatistics stats = new ArchiveStatistics();
614
615        int nid = getUnsignedByte(header);
616
617        if (nid == NID.kArchiveProperties) {
618            sanityCheckArchiveProperties(header);
619            nid = getUnsignedByte(header);
620        }
621
622        if (nid == NID.kAdditionalStreamsInfo) {
623            throw new IOException("Additional streams unsupported");
624            //nid = getUnsignedByte(header);
625        }
626
627        if (nid == NID.kMainStreamsInfo) {
628            sanityCheckStreamsInfo(header, stats);
629            nid = getUnsignedByte(header);
630        }
631
632        if (nid == NID.kFilesInfo) {
633            sanityCheckFilesInfo(header, stats);
634            nid = getUnsignedByte(header);
635        }
636
637        if (nid != NID.kEnd) {
638            throw new IOException("Badly terminated header, found " + nid);
639        }
640
641        return stats;
642    }
643
644    private void readArchiveProperties(final ByteBuffer input) throws IOException {
645        // FIXME: the reference implementation just throws them away?
646        int nid =  getUnsignedByte(input);
647        while (nid != NID.kEnd) {
648            final long propertySize = readUint64(input);
649            final byte[] property = new byte[(int)propertySize];
650            get(input, property);
651            nid = getUnsignedByte(input);
652        }
653    }
654
655    private void sanityCheckArchiveProperties(final ByteBuffer header)
656        throws IOException {
657        int nid =  getUnsignedByte(header);
658        while (nid != NID.kEnd) {
659            final int propertySize =
660                assertFitsIntoNonNegativeInt("propertySize", readUint64(header));
661            if (skipBytesFully(header, propertySize) < propertySize) {
662                throw new IOException("invalid property size");
663            }
664            nid = getUnsignedByte(header);
665        }
666    }
667
668    private ByteBuffer readEncodedHeader(final ByteBuffer header, final Archive archive,
669                                         final byte[] password) throws IOException {
670        final int pos = header.position();
671        ArchiveStatistics stats = new ArchiveStatistics();
672        sanityCheckStreamsInfo(header, stats);
673        stats.assertValidity(options.getMaxMemoryLimitInKb());
674        header.position(pos);
675
676        readStreamsInfo(header, archive);
677
678        if (archive.folders == null || archive.folders.length == 0) {
679            throw new IOException("no folders, can't read encoded header");
680        }
681        if (archive.packSizes == null || archive.packSizes.length == 0) {
682            throw new IOException("no packed streams, can't read encoded header");
683        }
684
685        // FIXME: merge with buildDecodingStream()/buildDecoderStack() at some stage?
686        final Folder folder = archive.folders[0];
687        final int firstPackStreamIndex = 0;
688        final long folderOffset = SIGNATURE_HEADER_SIZE + archive.packPos +
689                0;
690
691        channel.position(folderOffset);
692        InputStream inputStreamStack = new BoundedSeekableByteChannelInputStream(channel,
693                archive.packSizes[firstPackStreamIndex]);
694        for (final Coder coder : folder.getOrderedCoders()) {
695            if (coder.numInStreams != 1 || coder.numOutStreams != 1) {
696                throw new IOException("Multi input/output stream coders are not yet supported");
697            }
698            inputStreamStack = Coders.addDecoder(fileName, inputStreamStack, //NOSONAR
699                    folder.getUnpackSizeForCoder(coder), coder, password, options.getMaxMemoryLimitInKb());
700        }
701        if (folder.hasCrc) {
702            inputStreamStack = new CRC32VerifyingInputStream(inputStreamStack,
703                    folder.getUnpackSize(), folder.crc);
704        }
705        final int unpackSize = assertFitsIntoNonNegativeInt("unpackSize", folder.getUnpackSize());
706        final byte[] nextHeader = IOUtils.readRange(inputStreamStack, unpackSize);
707        if (nextHeader.length < unpackSize) {
708            throw new IOException("premature end of stream");
709        }
710        inputStreamStack.close();
711        return ByteBuffer.wrap(nextHeader).order(ByteOrder.LITTLE_ENDIAN);
712    }
713
714    private void sanityCheckStreamsInfo(final ByteBuffer header,
715        final ArchiveStatistics stats) throws IOException {
716        int nid = getUnsignedByte(header);
717
718        if (nid == NID.kPackInfo) {
719            sanityCheckPackInfo(header, stats);
720            nid = getUnsignedByte(header);
721        }
722
723        if (nid == NID.kUnpackInfo) {
724            sanityCheckUnpackInfo(header, stats);
725            nid = getUnsignedByte(header);
726        }
727
728        if (nid == NID.kSubStreamsInfo) {
729            sanityCheckSubStreamsInfo(header, stats);
730            nid = getUnsignedByte(header);
731        }
732
733        if (nid != NID.kEnd) {
734            throw new IOException("Badly terminated StreamsInfo");
735        }
736    }
737
738    private void readStreamsInfo(final ByteBuffer header, final Archive archive) throws IOException {
739        int nid = getUnsignedByte(header);
740
741        if (nid == NID.kPackInfo) {
742            readPackInfo(header, archive);
743            nid = getUnsignedByte(header);
744        }
745
746        if (nid == NID.kUnpackInfo) {
747            readUnpackInfo(header, archive);
748            nid = getUnsignedByte(header);
749        } else {
750            // archive without unpack/coders info
751            archive.folders = Folder.EMPTY_FOLDER_ARRAY;
752        }
753
754        if (nid == NID.kSubStreamsInfo) {
755            readSubStreamsInfo(header, archive);
756            nid = getUnsignedByte(header);
757        }
758    }
759
760    private void sanityCheckPackInfo(final ByteBuffer header, final ArchiveStatistics stats) throws IOException {
761        final long packPos = readUint64(header);
762        if (packPos < 0 || SIGNATURE_HEADER_SIZE + packPos > channel.size()
763            || SIGNATURE_HEADER_SIZE + packPos < 0) {
764            throw new IOException("packPos (" + packPos + ") is out of range");
765        }
766        final long numPackStreams = readUint64(header);
767        stats.numberOfPackedStreams = assertFitsIntoNonNegativeInt("numPackStreams", numPackStreams);
768        int nid = getUnsignedByte(header);
769        if (nid == NID.kSize) {
770            long totalPackSizes = 0;
771            for (int i = 0; i < stats.numberOfPackedStreams; i++) {
772                final long packSize = readUint64(header);
773                totalPackSizes += packSize;
774                final long endOfPackStreams = SIGNATURE_HEADER_SIZE + packPos + totalPackSizes;
775                if (packSize < 0
776                    || endOfPackStreams > channel.size()
777                    || endOfPackStreams < packPos) {
778                    throw new IOException("packSize (" + packSize + ") is out of range");
779                }
780            }
781            nid = getUnsignedByte(header);
782        }
783
784        if (nid == NID.kCRC) {
785            final int crcsDefined = readAllOrBits(header, stats.numberOfPackedStreams)
786                .cardinality();
787            if (skipBytesFully(header, 4 * crcsDefined) < 4 * crcsDefined) {
788                throw new IOException("invalid number of CRCs in PackInfo");
789            }
790            nid = getUnsignedByte(header);
791        }
792
793        if (nid != NID.kEnd) {
794            throw new IOException("Badly terminated PackInfo (" + nid + ")");
795        }
796    }
797
798    private void readPackInfo(final ByteBuffer header, final Archive archive) throws IOException {
799        archive.packPos = readUint64(header);
800        final int numPackStreamsInt = (int) readUint64(header);
801        int nid = getUnsignedByte(header);
802        if (nid == NID.kSize) {
803            archive.packSizes = new long[numPackStreamsInt];
804            for (int i = 0; i < archive.packSizes.length; i++) {
805                archive.packSizes[i] = readUint64(header);
806            }
807            nid = getUnsignedByte(header);
808        }
809
810        if (nid == NID.kCRC) {
811            archive.packCrcsDefined = readAllOrBits(header, numPackStreamsInt);
812            archive.packCrcs = new long[numPackStreamsInt];
813            for (int i = 0; i < numPackStreamsInt; i++) {
814                if (archive.packCrcsDefined.get(i)) {
815                    archive.packCrcs[i] = 0xffffFFFFL & getInt(header);
816                }
817            }
818
819            nid = getUnsignedByte(header);
820        }
821    }
822
823    private void sanityCheckUnpackInfo(final ByteBuffer header, final ArchiveStatistics stats)
824        throws IOException {
825        int nid = getUnsignedByte(header);
826        if (nid != NID.kFolder) {
827            throw new IOException("Expected kFolder, got " + nid);
828        }
829        final long numFolders = readUint64(header);
830        stats.numberOfFolders = assertFitsIntoNonNegativeInt("numFolders", numFolders);
831        final int external = getUnsignedByte(header);
832        if (external != 0) {
833            throw new IOException("External unsupported");
834        }
835
836        final List<Integer> numberOfOutputStreamsPerFolder = new LinkedList<>();
837        for (int i = 0; i < stats.numberOfFolders; i++) {
838            numberOfOutputStreamsPerFolder.add(sanityCheckFolder(header, stats));
839        }
840
841        final long totalNumberOfBindPairs = stats.numberOfOutStreams - stats.numberOfFolders;
842        final long packedStreamsRequiredByFolders = stats.numberOfInStreams - totalNumberOfBindPairs;
843        if (packedStreamsRequiredByFolders < stats.numberOfPackedStreams) {
844            throw new IOException("archive doesn't contain enough packed streams");
845        }
846
847        nid = getUnsignedByte(header);
848        if (nid != NID.kCodersUnpackSize) {
849            throw new IOException("Expected kCodersUnpackSize, got " + nid);
850        }
851
852        for (int numberOfOutputStreams : numberOfOutputStreamsPerFolder) {
853            for (int i = 0; i < numberOfOutputStreams; i++) {
854                final long unpackSize = readUint64(header);
855                if (unpackSize < 0) {
856                    throw new IllegalArgumentException("negative unpackSize");
857                }
858            }
859        }
860
861        nid = getUnsignedByte(header);
862        if (nid == NID.kCRC) {
863            stats.folderHasCrc = readAllOrBits(header, stats.numberOfFolders);
864            final int crcsDefined = stats.folderHasCrc.cardinality();
865            if (skipBytesFully(header, 4 * crcsDefined) < 4 * crcsDefined) {
866                throw new IOException("invalid number of CRCs in UnpackInfo");
867            }
868            nid = getUnsignedByte(header);
869        }
870
871        if (nid != NID.kEnd) {
872            throw new IOException("Badly terminated UnpackInfo");
873        }
874    }
875
876    private void readUnpackInfo(final ByteBuffer header, final Archive archive) throws IOException {
877        int nid = getUnsignedByte(header);
878        final int numFoldersInt = (int) readUint64(header);
879        final Folder[] folders = new Folder[numFoldersInt];
880        archive.folders = folders;
881        /* final int external = */ getUnsignedByte(header);
882        for (int i = 0; i < numFoldersInt; i++) {
883            folders[i] = readFolder(header);
884        }
885
886        nid = getUnsignedByte(header);
887        for (final Folder folder : folders) {
888            assertFitsIntoNonNegativeInt("totalOutputStreams", folder.totalOutputStreams);
889            folder.unpackSizes = new long[(int)folder.totalOutputStreams];
890            for (int i = 0; i < folder.totalOutputStreams; i++) {
891                folder.unpackSizes[i] = readUint64(header);
892            }
893        }
894
895        nid = getUnsignedByte(header);
896        if (nid == NID.kCRC) {
897            final BitSet crcsDefined = readAllOrBits(header, numFoldersInt);
898            for (int i = 0; i < numFoldersInt; i++) {
899                if (crcsDefined.get(i)) {
900                    folders[i].hasCrc = true;
901                    folders[i].crc = 0xffffFFFFL & getInt(header);
902                } else {
903                    folders[i].hasCrc = false;
904                }
905            }
906
907            nid = getUnsignedByte(header);
908        }
909    }
910
911    private void sanityCheckSubStreamsInfo(final ByteBuffer header, final ArchiveStatistics stats) throws IOException {
912
913        int nid = getUnsignedByte(header);
914        final List<Integer> numUnpackSubStreamsPerFolder = new LinkedList<>();
915        if (nid == NID.kNumUnpackStream) {
916            for (int i = 0; i < stats.numberOfFolders; i++) {
917                numUnpackSubStreamsPerFolder.add(assertFitsIntoNonNegativeInt("numStreams", readUint64(header)));
918            }
919            stats.numberOfUnpackSubStreams = numUnpackSubStreamsPerFolder.stream().collect(Collectors.summingLong(Integer::longValue));
920            nid = getUnsignedByte(header);
921        } else {
922            stats.numberOfUnpackSubStreams = stats.numberOfFolders;
923        }
924
925        assertFitsIntoNonNegativeInt("totalUnpackStreams", stats.numberOfUnpackSubStreams);
926
927        if (nid == NID.kSize) {
928            for (final int numUnpackSubStreams : numUnpackSubStreamsPerFolder) {
929                if (numUnpackSubStreams == 0) {
930                    continue;
931                }
932                for (int i = 0; i < numUnpackSubStreams - 1; i++) {
933                    final long size = readUint64(header);
934                    if (size < 0) {
935                        throw new IOException("negative unpackSize");
936                    }
937                }
938            }
939            nid = getUnsignedByte(header);
940        }
941
942        int numDigests = 0;
943        if (numUnpackSubStreamsPerFolder.isEmpty()) {
944            numDigests = stats.folderHasCrc == null ? stats.numberOfFolders
945                : stats.numberOfFolders - stats.folderHasCrc.cardinality();
946        } else {
947            int folderIdx = 0;
948            for (final int numUnpackSubStreams : numUnpackSubStreamsPerFolder) {
949                if (numUnpackSubStreams != 1 || stats.folderHasCrc == null
950                    || !stats.folderHasCrc.get(folderIdx++)) {
951                    numDigests += numUnpackSubStreams;
952                }
953            }
954        }
955
956        if (nid == NID.kCRC) {
957            assertFitsIntoNonNegativeInt("numDigests", numDigests);
958            final int missingCrcs = readAllOrBits(header, numDigests)
959                .cardinality();
960            if (skipBytesFully(header, 4 * missingCrcs) < 4 * missingCrcs) {
961                throw new IOException("invalid number of missing CRCs in SubStreamInfo");
962            }
963            nid = getUnsignedByte(header);
964        }
965
966        if (nid != NID.kEnd) {
967            throw new IOException("Badly terminated SubStreamsInfo");
968        }
969    }
970
971    private void readSubStreamsInfo(final ByteBuffer header, final Archive archive) throws IOException {
972        for (final Folder folder : archive.folders) {
973            folder.numUnpackSubStreams = 1;
974        }
975        long unpackStreamsCount = archive.folders.length;
976
977        int nid = getUnsignedByte(header);
978        if (nid == NID.kNumUnpackStream) {
979            unpackStreamsCount = 0;
980            for (final Folder folder : archive.folders) {
981                final long numStreams = readUint64(header);
982                folder.numUnpackSubStreams = (int)numStreams;
983                unpackStreamsCount += numStreams;
984            }
985            nid = getUnsignedByte(header);
986        }
987
988        final int totalUnpackStreams = (int) unpackStreamsCount;
989        final SubStreamsInfo subStreamsInfo = new SubStreamsInfo();
990        subStreamsInfo.unpackSizes = new long[totalUnpackStreams];
991        subStreamsInfo.hasCrc = new BitSet(totalUnpackStreams);
992        subStreamsInfo.crcs = new long[totalUnpackStreams];
993
994        int nextUnpackStream = 0;
995        for (final Folder folder : archive.folders) {
996            if (folder.numUnpackSubStreams == 0) {
997                continue;
998            }
999            long sum = 0;
1000            if (nid == NID.kSize) {
1001                for (int i = 0; i < folder.numUnpackSubStreams - 1; i++) {
1002                    final long size = readUint64(header);
1003                    subStreamsInfo.unpackSizes[nextUnpackStream++] = size;
1004                    sum += size;
1005                }
1006            }
1007            if (sum > folder.getUnpackSize()) {
1008                throw new IOException("sum of unpack sizes of folder exceeds total unpack size");
1009            }
1010            subStreamsInfo.unpackSizes[nextUnpackStream++] = folder.getUnpackSize() - sum;
1011        }
1012        if (nid == NID.kSize) {
1013            nid = getUnsignedByte(header);
1014        }
1015
1016        int numDigests = 0;
1017        for (final Folder folder : archive.folders) {
1018            if (folder.numUnpackSubStreams != 1 || !folder.hasCrc) {
1019                numDigests += folder.numUnpackSubStreams;
1020            }
1021        }
1022
1023        if (nid == NID.kCRC) {
1024            final BitSet hasMissingCrc = readAllOrBits(header, numDigests);
1025            final long[] missingCrcs = new long[numDigests];
1026            for (int i = 0; i < numDigests; i++) {
1027                if (hasMissingCrc.get(i)) {
1028                    missingCrcs[i] = 0xffffFFFFL & getInt(header);
1029                }
1030            }
1031            int nextCrc = 0;
1032            int nextMissingCrc = 0;
1033            for (final Folder folder: archive.folders) {
1034                if (folder.numUnpackSubStreams == 1 && folder.hasCrc) {
1035                    subStreamsInfo.hasCrc.set(nextCrc, true);
1036                    subStreamsInfo.crcs[nextCrc] = folder.crc;
1037                    ++nextCrc;
1038                } else {
1039                    for (int i = 0; i < folder.numUnpackSubStreams; i++) {
1040                        subStreamsInfo.hasCrc.set(nextCrc, hasMissingCrc.get(nextMissingCrc));
1041                        subStreamsInfo.crcs[nextCrc] = missingCrcs[nextMissingCrc];
1042                        ++nextCrc;
1043                        ++nextMissingCrc;
1044                    }
1045                }
1046            }
1047
1048            nid = getUnsignedByte(header);
1049        }
1050
1051        archive.subStreamsInfo = subStreamsInfo;
1052    }
1053
1054    private int sanityCheckFolder(final ByteBuffer header, final ArchiveStatistics stats)
1055        throws IOException {
1056
1057        final int numCoders = assertFitsIntoNonNegativeInt("numCoders", readUint64(header));
1058        if (numCoders == 0) {
1059            throw new IOException("Folder without coders");
1060        }
1061        stats.numberOfCoders += numCoders;
1062
1063        long totalOutStreams = 0;
1064        long totalInStreams = 0;
1065        for (int i = 0; i < numCoders; i++) {
1066            final int bits = getUnsignedByte(header);
1067            final int idSize = bits & 0xf;
1068            get(header, new byte[idSize]);
1069
1070            final boolean isSimple = (bits & 0x10) == 0;
1071            final boolean hasAttributes = (bits & 0x20) != 0;
1072            final boolean moreAlternativeMethods = (bits & 0x80) != 0;
1073            if (moreAlternativeMethods) {
1074                throw new IOException("Alternative methods are unsupported, please report. " + // NOSONAR
1075                    "The reference implementation doesn't support them either.");
1076            }
1077
1078            if (isSimple) {
1079                totalInStreams++;
1080                totalOutStreams++;
1081            } else {
1082                totalInStreams +=
1083                    assertFitsIntoNonNegativeInt("numInStreams", readUint64(header));
1084                totalOutStreams +=
1085                    assertFitsIntoNonNegativeInt("numOutStreams", readUint64(header));
1086            }
1087
1088            if (hasAttributes) {
1089                final int propertiesSize =
1090                    assertFitsIntoNonNegativeInt("propertiesSize", readUint64(header));
1091                if (skipBytesFully(header, propertiesSize) < propertiesSize) {
1092                    throw new IOException("invalid propertiesSize in folder");
1093                }
1094            }
1095        }
1096        assertFitsIntoNonNegativeInt("totalInStreams", totalInStreams);
1097        assertFitsIntoNonNegativeInt("totalOutStreams", totalOutStreams);
1098        stats.numberOfOutStreams += totalOutStreams;
1099        stats.numberOfInStreams += totalInStreams;
1100
1101        if (totalOutStreams == 0) {
1102            throw new IOException("Total output streams can't be 0");
1103        }
1104
1105        final int numBindPairs =
1106            assertFitsIntoNonNegativeInt("numBindPairs", totalOutStreams - 1);
1107        if (totalInStreams < numBindPairs) {
1108            throw new IOException("Total input streams can't be less than the number of bind pairs");
1109        }
1110        final BitSet inStreamsBound = new BitSet((int) totalInStreams);
1111        for (int i = 0; i < numBindPairs; i++) {
1112            final int inIndex = assertFitsIntoNonNegativeInt("inIndex", readUint64(header));
1113            if (totalInStreams <= inIndex) {
1114                throw new IOException("inIndex is bigger than number of inStreams");
1115            }
1116            inStreamsBound.set(inIndex);
1117            final int outIndex = assertFitsIntoNonNegativeInt("outIndex", readUint64(header));
1118            if (totalOutStreams <= outIndex) {
1119                throw new IOException("outIndex is bigger than number of outStreams");
1120            }
1121        }
1122
1123        final int numPackedStreams =
1124            assertFitsIntoNonNegativeInt("numPackedStreams", totalInStreams - numBindPairs);
1125
1126        if (numPackedStreams == 1) {
1127            if (inStreamsBound.nextClearBit(0) == -1) {
1128                throw new IOException("Couldn't find stream's bind pair index");
1129            }
1130        } else {
1131            for (int i = 0; i < numPackedStreams; i++) {
1132                final int packedStreamIndex =
1133                    assertFitsIntoNonNegativeInt("packedStreamIndex", readUint64(header));
1134                if (packedStreamIndex >= totalInStreams) {
1135                    throw new IOException("packedStreamIndex is bigger than number of totalInStreams");
1136                }
1137            }
1138        }
1139
1140        return (int) totalOutStreams;
1141    }
1142
1143    private Folder readFolder(final ByteBuffer header) throws IOException {
1144        final Folder folder = new Folder();
1145
1146        final long numCoders = readUint64(header);
1147        final Coder[] coders = new Coder[(int)numCoders];
1148        long totalInStreams = 0;
1149        long totalOutStreams = 0;
1150        for (int i = 0; i < coders.length; i++) {
1151            coders[i] = new Coder();
1152            final int bits = getUnsignedByte(header);
1153            final int idSize = bits & 0xf;
1154            final boolean isSimple = (bits & 0x10) == 0;
1155            final boolean hasAttributes = (bits & 0x20) != 0;
1156            final boolean moreAlternativeMethods = (bits & 0x80) != 0;
1157
1158            coders[i].decompressionMethodId = new byte[idSize];
1159            get(header, coders[i].decompressionMethodId);
1160            if (isSimple) {
1161                coders[i].numInStreams = 1;
1162                coders[i].numOutStreams = 1;
1163            } else {
1164                coders[i].numInStreams = readUint64(header);
1165                coders[i].numOutStreams = readUint64(header);
1166            }
1167            totalInStreams += coders[i].numInStreams;
1168            totalOutStreams += coders[i].numOutStreams;
1169            if (hasAttributes) {
1170                final long propertiesSize = readUint64(header);
1171                coders[i].properties = new byte[(int)propertiesSize];
1172                get(header, coders[i].properties);
1173            }
1174            // would need to keep looping as above:
1175            while (moreAlternativeMethods) {
1176                throw new IOException("Alternative methods are unsupported, please report. " + // NOSONAR
1177                    "The reference implementation doesn't support them either.");
1178            }
1179        }
1180        folder.coders = coders;
1181        folder.totalInputStreams = totalInStreams;
1182        folder.totalOutputStreams = totalOutStreams;
1183
1184        final long numBindPairs = totalOutStreams - 1;
1185        final BindPair[] bindPairs = new BindPair[(int)numBindPairs];
1186        for (int i = 0; i < bindPairs.length; i++) {
1187            bindPairs[i] = new BindPair();
1188            bindPairs[i].inIndex = readUint64(header);
1189            bindPairs[i].outIndex = readUint64(header);
1190        }
1191        folder.bindPairs = bindPairs;
1192
1193        final long numPackedStreams = totalInStreams - numBindPairs;
1194        final long[] packedStreams = new long[(int)numPackedStreams];
1195        if (numPackedStreams == 1) {
1196            int i;
1197            for (i = 0; i < (int)totalInStreams; i++) {
1198                if (folder.findBindPairForInStream(i) < 0) {
1199                    break;
1200                }
1201            }
1202            packedStreams[0] = i;
1203        } else {
1204            for (int i = 0; i < (int)numPackedStreams; i++) {
1205                packedStreams[i] = readUint64(header);
1206            }
1207        }
1208        folder.packedStreams = packedStreams;
1209
1210        return folder;
1211    }
1212
1213    private BitSet readAllOrBits(final ByteBuffer header, final int size) throws IOException {
1214        final int areAllDefined = getUnsignedByte(header);
1215        final BitSet bits;
1216        if (areAllDefined != 0) {
1217            bits = new BitSet(size);
1218            for (int i = 0; i < size; i++) {
1219                bits.set(i, true);
1220            }
1221        } else {
1222            bits = readBits(header, size);
1223        }
1224        return bits;
1225    }
1226
1227    private BitSet readBits(final ByteBuffer header, final int size) throws IOException {
1228        final BitSet bits = new BitSet(size);
1229        int mask = 0;
1230        int cache = 0;
1231        for (int i = 0; i < size; i++) {
1232            if (mask == 0) {
1233                mask = 0x80;
1234                cache = getUnsignedByte(header);
1235            }
1236            bits.set(i, (cache & mask) != 0);
1237            mask >>>= 1;
1238        }
1239        return bits;
1240    }
1241
1242    private void sanityCheckFilesInfo(final ByteBuffer header, final ArchiveStatistics stats) throws IOException {
1243        stats.numberOfEntries = assertFitsIntoNonNegativeInt("numFiles", readUint64(header));
1244
1245        int emptyStreams = -1;
1246        while (true) {
1247            final int propertyType = getUnsignedByte(header);
1248            if (propertyType == 0) {
1249                break;
1250            }
1251            final long size = readUint64(header);
1252            switch (propertyType) {
1253                case NID.kEmptyStream: {
1254                    emptyStreams = readBits(header, stats.numberOfEntries).cardinality();
1255                    break;
1256                }
1257                case NID.kEmptyFile: {
1258                    if (emptyStreams == -1) {
1259                        throw new IOException("Header format error: kEmptyStream must appear before kEmptyFile");
1260                    }
1261                    readBits(header, emptyStreams);
1262                    break;
1263                }
1264                case NID.kAnti: {
1265                    if (emptyStreams == -1) {
1266                        throw new IOException("Header format error: kEmptyStream must appear before kAnti");
1267                    }
1268                    readBits(header, emptyStreams);
1269                    break;
1270                }
1271                case NID.kName: {
1272                    final int external = getUnsignedByte(header);
1273                    if (external != 0) {
1274                        throw new IOException("Not implemented");
1275                    }
1276                    final int namesLength =
1277                        assertFitsIntoNonNegativeInt("file names length", size - 1);
1278                    if ((namesLength & 1) != 0) {
1279                        throw new IOException("File names length invalid");
1280                    }
1281
1282                    int filesSeen = 0;
1283                    for (int i = 0; i < namesLength; i += 2) {
1284                        final char c = getChar(header);
1285                        if (c == 0) {
1286                            filesSeen++;
1287                        }
1288                    }
1289                    if (filesSeen != stats.numberOfEntries) {
1290                        throw new IOException("Invalid number of file names (" + filesSeen + " instead of "
1291                            + stats.numberOfEntries + ")");
1292                    }
1293                    break;
1294                }
1295                case NID.kCTime: {
1296                    final int timesDefined = readAllOrBits(header, stats.numberOfEntries)
1297                        .cardinality();
1298                    final int external = getUnsignedByte(header);
1299                    if (external != 0) {
1300                        throw new IOException("Not implemented");
1301                    }
1302                    if (skipBytesFully(header, 8 * timesDefined) < 8 * timesDefined) {
1303                        throw new IOException("invalid creation dates size");
1304                    }
1305                    break;
1306                }
1307                case NID.kATime: {
1308                    final int timesDefined = readAllOrBits(header, stats.numberOfEntries)
1309                        .cardinality();
1310                    final int external = getUnsignedByte(header);
1311                    if (external != 0) {
1312                        throw new IOException("Not implemented");
1313                    }
1314                    if (skipBytesFully(header, 8 * timesDefined) < 8 * timesDefined) {
1315                        throw new IOException("invalid access dates size");
1316                    }
1317                    break;
1318                }
1319                case NID.kMTime: {
1320                    final int timesDefined = readAllOrBits(header, stats.numberOfEntries)
1321                        .cardinality();
1322                    final int external = getUnsignedByte(header);
1323                    if (external != 0) {
1324                        throw new IOException("Not implemented");
1325                    }
1326                    if (skipBytesFully(header, 8 * timesDefined) < 8 * timesDefined) {
1327                        throw new IOException("invalid modification dates size");
1328                    }
1329                    break;
1330                }
1331                case NID.kWinAttributes: {
1332                    final int attributesDefined = readAllOrBits(header, stats.numberOfEntries)
1333                        .cardinality();
1334                    final int external = getUnsignedByte(header);
1335                    if (external != 0) {
1336                        throw new IOException("Not implemented");
1337                    }
1338                    if (skipBytesFully(header, 4 * attributesDefined) < 4 * attributesDefined) {
1339                        throw new IOException("invalid windows attributes size");
1340                    }
1341                    break;
1342                }
1343                case NID.kStartPos: {
1344                    throw new IOException("kStartPos is unsupported, please report");
1345                }
1346                case NID.kDummy: {
1347                    // 7z 9.20 asserts the content is all zeros and ignores the property
1348                    // Compress up to 1.8.1 would throw an exception, now we ignore it (see COMPRESS-287
1349
1350                    if (skipBytesFully(header, size) < size) {
1351                        throw new IOException("Incomplete kDummy property");
1352                    }
1353                    break;
1354                }
1355
1356                default: {
1357                    // Compress up to 1.8.1 would throw an exception, now we ignore it (see COMPRESS-287
1358                    if (skipBytesFully(header, size) < size) {
1359                        throw new IOException("Incomplete property of type " + propertyType);
1360                    }
1361                    break;
1362                }
1363            }
1364        }
1365        stats.numberOfEntriesWithStream = stats.numberOfEntries - (emptyStreams > 0 ? emptyStreams : 0);
1366    }
1367
1368    private void readFilesInfo(final ByteBuffer header, final Archive archive) throws IOException {
1369        final int numFilesInt = (int) readUint64(header);;
1370        final Map<Integer, SevenZArchiveEntry> fileMap = new HashMap<>();
1371        BitSet isEmptyStream = null;
1372        BitSet isEmptyFile = null;
1373        BitSet isAnti = null;
1374        while (true) {
1375            final int propertyType = getUnsignedByte(header);
1376            if (propertyType == 0) {
1377                break;
1378            }
1379            final long size = readUint64(header);
1380            switch (propertyType) {
1381                case NID.kEmptyStream: {
1382                    isEmptyStream = readBits(header, numFilesInt);
1383                    break;
1384                }
1385                case NID.kEmptyFile: {
1386                    isEmptyFile = readBits(header, isEmptyStream.cardinality());
1387                    break;
1388                }
1389                case NID.kAnti: {
1390                    isAnti = readBits(header, isEmptyStream.cardinality());
1391                    break;
1392                }
1393                case NID.kName: {
1394                    /* final int external = */ getUnsignedByte(header);
1395                    final byte[] names = new byte[(int) (size - 1)];
1396                    final int namesLength = names.length;
1397                    get(header, names);
1398                    int nextFile = 0;
1399                    int nextName = 0;
1400                    for (int i = 0; i < namesLength; i += 2) {
1401                        if (names[i] == 0 && names[i + 1] == 0) {
1402                            checkEntryIsInitialized(fileMap, nextFile);
1403                            fileMap.get(nextFile).setName(new String(names, nextName, i - nextName, StandardCharsets.UTF_16LE));
1404                            nextName = i + 2;
1405                            nextFile++;
1406                        }
1407                    }
1408                    if (nextName != namesLength || nextFile != numFilesInt) {
1409                        throw new IOException("Error parsing file names");
1410                    }
1411                    break;
1412                }
1413                case NID.kCTime: {
1414                    final BitSet timesDefined = readAllOrBits(header, numFilesInt);
1415                    /* final int external = */ getUnsignedByte(header);
1416                    for (int i = 0; i < numFilesInt; i++) {
1417                        checkEntryIsInitialized(fileMap, i);
1418                        final SevenZArchiveEntry entryAtIndex = fileMap.get(i);
1419                        entryAtIndex.setHasCreationDate(timesDefined.get(i));
1420                        if (entryAtIndex.getHasCreationDate()) {
1421                            entryAtIndex.setCreationDate(getLong(header));
1422                        }
1423                    }
1424                    break;
1425                }
1426                case NID.kATime: {
1427                    final BitSet timesDefined = readAllOrBits(header, numFilesInt);
1428                    /* final int external = */ getUnsignedByte(header);
1429                    for (int i = 0; i < numFilesInt; i++) {
1430                        checkEntryIsInitialized(fileMap, i);
1431                        final SevenZArchiveEntry entryAtIndex = fileMap.get(i);
1432                        entryAtIndex.setHasAccessDate(timesDefined.get(i));
1433                        if (entryAtIndex.getHasAccessDate()) {
1434                            entryAtIndex.setAccessDate(getLong(header));
1435                        }
1436                    }
1437                    break;
1438                }
1439                case NID.kMTime: {
1440                    final BitSet timesDefined = readAllOrBits(header, numFilesInt);
1441                    /* final int external = */ getUnsignedByte(header);
1442                    for (int i = 0; i < numFilesInt; i++) {
1443                        checkEntryIsInitialized(fileMap, i);
1444                        final SevenZArchiveEntry entryAtIndex = fileMap.get(i);
1445                        entryAtIndex.setHasLastModifiedDate(timesDefined.get(i));
1446                        if (entryAtIndex.getHasLastModifiedDate()) {
1447                            entryAtIndex.setLastModifiedDate(getLong(header));
1448                        }
1449                    }
1450                    break;
1451                }
1452                case NID.kWinAttributes: {
1453                    final BitSet attributesDefined = readAllOrBits(header, numFilesInt);
1454                    /* final int external = */ getUnsignedByte(header);
1455                    for (int i = 0; i < numFilesInt; i++) {
1456                        checkEntryIsInitialized(fileMap, i);
1457                        final SevenZArchiveEntry entryAtIndex = fileMap.get(i);
1458                        entryAtIndex.setHasWindowsAttributes(attributesDefined.get(i));
1459                        if (entryAtIndex.getHasWindowsAttributes()) {
1460                            entryAtIndex.setWindowsAttributes(getInt(header));
1461                        }
1462                    }
1463                    break;
1464                }
1465                case NID.kDummy: {
1466                    // 7z 9.20 asserts the content is all zeros and ignores the property
1467                    // Compress up to 1.8.1 would throw an exception, now we ignore it (see COMPRESS-287
1468
1469                    skipBytesFully(header, size);
1470                    break;
1471                }
1472
1473                default: {
1474                    // Compress up to 1.8.1 would throw an exception, now we ignore it (see COMPRESS-287
1475                    skipBytesFully(header, size);
1476                    break;
1477                }
1478            }
1479        }
1480        int nonEmptyFileCounter = 0;
1481        int emptyFileCounter = 0;
1482        for (int i = 0; i < numFilesInt; i++) {
1483            final SevenZArchiveEntry entryAtIndex = fileMap.get(i);
1484            if (entryAtIndex == null) {
1485                continue;
1486            }
1487            entryAtIndex.setHasStream(isEmptyStream == null || !isEmptyStream.get(i));
1488            if (entryAtIndex.hasStream()) {
1489                if (archive.subStreamsInfo == null) {
1490                    throw new IOException("Archive contains file with streams but no subStreamsInfo");
1491                }
1492                entryAtIndex.setDirectory(false);
1493                entryAtIndex.setAntiItem(false);
1494                entryAtIndex.setHasCrc(archive.subStreamsInfo.hasCrc.get(nonEmptyFileCounter));
1495                entryAtIndex.setCrcValue(archive.subStreamsInfo.crcs[nonEmptyFileCounter]);
1496                entryAtIndex.setSize(archive.subStreamsInfo.unpackSizes[nonEmptyFileCounter]);
1497                if (entryAtIndex.getSize() < 0) {
1498                    throw new IOException("broken archive, entry with negative size");
1499                }
1500                ++nonEmptyFileCounter;
1501            } else {
1502                entryAtIndex.setDirectory(isEmptyFile == null || !isEmptyFile.get(emptyFileCounter));
1503                entryAtIndex.setAntiItem(isAnti != null && isAnti.get(emptyFileCounter));
1504                entryAtIndex.setHasCrc(false);
1505                entryAtIndex.setSize(0);
1506                ++emptyFileCounter;
1507            }
1508        }
1509        final List<SevenZArchiveEntry> entries = new ArrayList<>();
1510        for (final SevenZArchiveEntry e : fileMap.values()) {
1511            if (e != null) {
1512                entries.add(e);
1513            }
1514        }
1515        archive.files = entries.toArray(SevenZArchiveEntry.EMPTY_SEVEN_Z_ARCHIVE_ENTRY_ARRAY);
1516        calculateStreamMap(archive);
1517    }
1518
1519    private void checkEntryIsInitialized(final Map<Integer, SevenZArchiveEntry> archiveEntries, final int index) {
1520        if (archiveEntries.get(index) == null) {
1521            archiveEntries.put(index, new SevenZArchiveEntry());
1522        }
1523    }
1524
1525    private void calculateStreamMap(final Archive archive) throws IOException {
1526        final StreamMap streamMap = new StreamMap();
1527
1528        int nextFolderPackStreamIndex = 0;
1529        final int numFolders = archive.folders != null ? archive.folders.length : 0;
1530        streamMap.folderFirstPackStreamIndex = new int[numFolders];
1531        for (int i = 0; i < numFolders; i++) {
1532            streamMap.folderFirstPackStreamIndex[i] = nextFolderPackStreamIndex;
1533            nextFolderPackStreamIndex += archive.folders[i].packedStreams.length;
1534        }
1535
1536        long nextPackStreamOffset = 0;
1537        final int numPackSizes = archive.packSizes.length;
1538        streamMap.packStreamOffsets = new long[numPackSizes];
1539        for (int i = 0; i < numPackSizes; i++) {
1540            streamMap.packStreamOffsets[i] = nextPackStreamOffset;
1541            nextPackStreamOffset += archive.packSizes[i];
1542        }
1543
1544        streamMap.folderFirstFileIndex = new int[numFolders];
1545        streamMap.fileFolderIndex = new int[archive.files.length];
1546        int nextFolderIndex = 0;
1547        int nextFolderUnpackStreamIndex = 0;
1548        for (int i = 0; i < archive.files.length; i++) {
1549            if (!archive.files[i].hasStream() && nextFolderUnpackStreamIndex == 0) {
1550                streamMap.fileFolderIndex[i] = -1;
1551                continue;
1552            }
1553            if (nextFolderUnpackStreamIndex == 0) {
1554                for (; nextFolderIndex < archive.folders.length; ++nextFolderIndex) {
1555                    streamMap.folderFirstFileIndex[nextFolderIndex] = i;
1556                    if (archive.folders[nextFolderIndex].numUnpackSubStreams > 0) {
1557                        break;
1558                    }
1559                }
1560                if (nextFolderIndex >= archive.folders.length) {
1561                    throw new IOException("Too few folders in archive");
1562                }
1563            }
1564            streamMap.fileFolderIndex[i] = nextFolderIndex;
1565            if (!archive.files[i].hasStream()) {
1566                continue;
1567            }
1568            ++nextFolderUnpackStreamIndex;
1569            if (nextFolderUnpackStreamIndex >= archive.folders[nextFolderIndex].numUnpackSubStreams) {
1570                ++nextFolderIndex;
1571                nextFolderUnpackStreamIndex = 0;
1572            }
1573        }
1574
1575        archive.streamMap = streamMap;
1576    }
1577
1578    /**
1579     * Build the decoding stream for the entry to be read.
1580     * This method may be called from a random access(getInputStream) or
1581     * sequential access(getNextEntry).
1582     * If this method is called from a random access, some entries may
1583     * need to be skipped(we put them to the deferredBlockStreams and
1584     * skip them when actually needed to improve the performance)
1585     *
1586     * @param entryIndex     the index of the entry to be read
1587     * @param isRandomAccess is this called in a random access
1588     * @throws IOException if there are exceptions when reading the file
1589     */
1590    private void buildDecodingStream(final int entryIndex, final boolean isRandomAccess) throws IOException {
1591        if (archive.streamMap == null) {
1592            throw new IOException("Archive doesn't contain stream information to read entries");
1593        }
1594        final int folderIndex = archive.streamMap.fileFolderIndex[entryIndex];
1595        if (folderIndex < 0) {
1596            deferredBlockStreams.clear();
1597            // TODO: previously it'd return an empty stream?
1598            // new BoundedInputStream(new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY), 0);
1599            return;
1600        }
1601        final SevenZArchiveEntry file = archive.files[entryIndex];
1602        boolean isInSameFolder = false;
1603        if (currentFolderIndex == folderIndex) {
1604            // (COMPRESS-320).
1605            // The current entry is within the same (potentially opened) folder. The
1606            // previous stream has to be fully decoded before we can start reading
1607            // but don't do it eagerly -- if the user skips over the entire folder nothing
1608            // is effectively decompressed.
1609            if (entryIndex > 0) {
1610                file.setContentMethods(archive.files[entryIndex - 1].getContentMethods());
1611            }
1612
1613            // if this is called in a random access, then the content methods of previous entry may be null
1614            // the content methods should be set to methods of the first entry as it must not be null,
1615            // and the content methods would only be set if the content methods was not set
1616            if(isRandomAccess && file.getContentMethods() == null) {
1617                final int folderFirstFileIndex = archive.streamMap.folderFirstFileIndex[folderIndex];
1618                final SevenZArchiveEntry folderFirstFile = archive.files[folderFirstFileIndex];
1619                file.setContentMethods(folderFirstFile.getContentMethods());
1620            }
1621            isInSameFolder = true;
1622        } else {
1623            currentFolderIndex = folderIndex;
1624            // We're opening a new folder. Discard any queued streams/ folder stream.
1625            reopenFolderInputStream(folderIndex, file);
1626        }
1627
1628        boolean haveSkippedEntries = false;
1629        if (isRandomAccess) {
1630            // entries will only need to be skipped if it's a random access
1631            haveSkippedEntries = skipEntriesWhenNeeded(entryIndex, isInSameFolder, folderIndex);
1632        }
1633
1634        if (isRandomAccess && currentEntryIndex == entryIndex && !haveSkippedEntries) {
1635            // we don't need to add another entry to the deferredBlockStreams when :
1636            // 1. If this method is called in a random access and the entry index
1637            // to be read equals to the current entry index, the input stream
1638            // has already been put in the deferredBlockStreams
1639            // 2. If this entry has not been read(which means no entries are skipped)
1640            return;
1641        }
1642
1643        InputStream fileStream = new BoundedInputStream(currentFolderInputStream, file.getSize());
1644        if (file.getHasCrc()) {
1645            fileStream = new CRC32VerifyingInputStream(fileStream, file.getSize(), file.getCrcValue());
1646        }
1647
1648        deferredBlockStreams.add(fileStream);
1649    }
1650
1651    /**
1652     * Discard any queued streams/ folder stream, and reopen the current folder input stream.
1653     *
1654     * @param folderIndex the index of the folder to reopen
1655     * @param file        the 7z entry to read
1656     * @throws IOException if exceptions occur when reading the 7z file
1657     */
1658    private void reopenFolderInputStream(final int folderIndex, final SevenZArchiveEntry file) throws IOException {
1659        deferredBlockStreams.clear();
1660        if (currentFolderInputStream != null) {
1661            currentFolderInputStream.close();
1662            currentFolderInputStream = null;
1663        }
1664        final Folder folder = archive.folders[folderIndex];
1665        final int firstPackStreamIndex = archive.streamMap.folderFirstPackStreamIndex[folderIndex];
1666        final long folderOffset = SIGNATURE_HEADER_SIZE + archive.packPos +
1667                archive.streamMap.packStreamOffsets[firstPackStreamIndex];
1668
1669        currentFolderInputStream = buildDecoderStack(folder, folderOffset, firstPackStreamIndex, file);
1670    }
1671
1672    /**
1673     * Skip all the entries if needed.
1674     * Entries need to be skipped when:
1675     * <p>
1676     * 1. it's a random access
1677     * 2. one of these 2 condition is meet :
1678     * <p>
1679     * 2.1 currentEntryIndex != entryIndex : this means there are some entries
1680     * to be skipped(currentEntryIndex < entryIndex) or the entry has already
1681     * been read(currentEntryIndex > entryIndex)
1682     * <p>
1683     * 2.2 currentEntryIndex == entryIndex && !hasCurrentEntryBeenRead:
1684     * if the entry to be read is the current entry, but some data of it has
1685     * been read before, then we need to reopen the stream of the folder and
1686     * skip all the entries before the current entries
1687     *
1688     * @param entryIndex     the entry to be read
1689     * @param isInSameFolder are the entry to be read and the current entry in the same folder
1690     * @param folderIndex    the index of the folder which contains the entry
1691     * @return true if there are entries actually skipped
1692     * @throws IOException there are exceptions when skipping entries
1693     * @since 1.21
1694     */
1695    private boolean skipEntriesWhenNeeded(final int entryIndex, final boolean isInSameFolder, final int folderIndex) throws IOException {
1696        final SevenZArchiveEntry file = archive.files[entryIndex];
1697        // if the entry to be read is the current entry, and the entry has not
1698        // been read yet, then there's nothing we need to do
1699        if (currentEntryIndex == entryIndex && !hasCurrentEntryBeenRead()) {
1700            return false;
1701        }
1702
1703        // 1. if currentEntryIndex < entryIndex :
1704        // this means there are some entries to be skipped(currentEntryIndex < entryIndex)
1705        // 2. if currentEntryIndex > entryIndex || (currentEntryIndex == entryIndex && hasCurrentEntryBeenRead) :
1706        // this means the entry has already been read before, and we need to reopen the
1707        // stream of the folder and skip all the entries before the current entries
1708        int filesToSkipStartIndex = archive.streamMap.folderFirstFileIndex[currentFolderIndex];
1709        if (isInSameFolder) {
1710            if (currentEntryIndex < entryIndex) {
1711                // the entries between filesToSkipStartIndex and currentEntryIndex had already been skipped
1712                filesToSkipStartIndex = currentEntryIndex + 1;
1713            } else {
1714                // the entry is in the same folder of current entry, but it has already been read before, we need to reset
1715                // the position of the currentFolderInputStream to the beginning of folder, and then skip the files
1716                // from the start entry of the folder again
1717                reopenFolderInputStream(folderIndex, file);
1718            }
1719        }
1720
1721        for (int i = filesToSkipStartIndex; i < entryIndex; i++) {
1722            final SevenZArchiveEntry fileToSkip = archive.files[i];
1723            InputStream fileStreamToSkip = new BoundedInputStream(currentFolderInputStream, fileToSkip.getSize());
1724            if (fileToSkip.getHasCrc()) {
1725                fileStreamToSkip = new CRC32VerifyingInputStream(fileStreamToSkip, fileToSkip.getSize(), fileToSkip.getCrcValue());
1726            }
1727            deferredBlockStreams.add(fileStreamToSkip);
1728
1729            // set the content methods as well, it equals to file.getContentMethods() because they are in same folder
1730            fileToSkip.setContentMethods(file.getContentMethods());
1731        }
1732        return true;
1733    }
1734
1735    /**
1736     * Find out if any data of current entry has been read or not.
1737     * This is achieved by comparing the bytes remaining to read
1738     * and the size of the file.
1739     *
1740     * @return true if any data of current entry has been read
1741     * @since 1.21
1742     */
1743    private boolean hasCurrentEntryBeenRead() {
1744        boolean hasCurrentEntryBeenRead = false;
1745        if (!deferredBlockStreams.isEmpty()) {
1746            final InputStream currentEntryInputStream = deferredBlockStreams.get(deferredBlockStreams.size() - 1);
1747            // get the bytes remaining to read, and compare it with the size of
1748            // the file to figure out if the file has been read
1749            if (currentEntryInputStream instanceof CRC32VerifyingInputStream) {
1750                hasCurrentEntryBeenRead = ((CRC32VerifyingInputStream) currentEntryInputStream).getBytesRemaining() != archive.files[currentEntryIndex].getSize();
1751            }
1752
1753            if (currentEntryInputStream instanceof BoundedInputStream) {
1754                hasCurrentEntryBeenRead = ((BoundedInputStream) currentEntryInputStream).getBytesRemaining() != archive.files[currentEntryIndex].getSize();
1755            }
1756        }
1757        return hasCurrentEntryBeenRead;
1758    }
1759
1760    private InputStream buildDecoderStack(final Folder folder, final long folderOffset,
1761                final int firstPackStreamIndex, final SevenZArchiveEntry entry) throws IOException {
1762        channel.position(folderOffset);
1763        InputStream inputStreamStack = new FilterInputStream(new BufferedInputStream(
1764              new BoundedSeekableByteChannelInputStream(channel,
1765                  archive.packSizes[firstPackStreamIndex]))) {
1766            @Override
1767            public int read() throws IOException {
1768                final int r = in.read();
1769                if (r >= 0) {
1770                    count(1);
1771                }
1772                return r;
1773            }
1774            @Override
1775            public int read(final byte[] b) throws IOException {
1776                return read(b, 0, b.length);
1777            }
1778            @Override
1779            public int read(final byte[] b, final int off, final int len) throws IOException {
1780                if (len == 0) {
1781                    return 0;
1782                }
1783                final int r = in.read(b, off, len);
1784                if (r >= 0) {
1785                    count(r);
1786                }
1787                return r;
1788            }
1789            private void count(final int c) {
1790                compressedBytesReadFromCurrentEntry += c;
1791            }
1792        };
1793        final LinkedList<SevenZMethodConfiguration> methods = new LinkedList<>();
1794        for (final Coder coder : folder.getOrderedCoders()) {
1795            if (coder.numInStreams != 1 || coder.numOutStreams != 1) {
1796                throw new IOException("Multi input/output stream coders are not yet supported");
1797            }
1798            final SevenZMethod method = SevenZMethod.byId(coder.decompressionMethodId);
1799            inputStreamStack = Coders.addDecoder(fileName, inputStreamStack,
1800                    folder.getUnpackSizeForCoder(coder), coder, password, options.getMaxMemoryLimitInKb());
1801            methods.addFirst(new SevenZMethodConfiguration(method,
1802                     Coders.findByMethod(method).getOptionsFromCoder(coder, inputStreamStack)));
1803        }
1804        entry.setContentMethods(methods);
1805        if (folder.hasCrc) {
1806            return new CRC32VerifyingInputStream(inputStreamStack,
1807                    folder.getUnpackSize(), folder.crc);
1808        }
1809        return inputStreamStack;
1810    }
1811
1812    /**
1813     * Reads a byte of data.
1814     *
1815     * @return the byte read, or -1 if end of input is reached
1816     * @throws IOException
1817     *             if an I/O error has occurred
1818     */
1819    public int read() throws IOException {
1820        final int b = getCurrentStream().read();
1821        if (b >= 0) {
1822            uncompressedBytesReadFromCurrentEntry++;
1823        }
1824        return b;
1825    }
1826
1827    private InputStream getCurrentStream() throws IOException {
1828        if (archive.files[currentEntryIndex].getSize() == 0) {
1829            return new ByteArrayInputStream(ByteUtils.EMPTY_BYTE_ARRAY);
1830        }
1831        if (deferredBlockStreams.isEmpty()) {
1832            throw new IllegalStateException("No current 7z entry (call getNextEntry() first).");
1833        }
1834
1835        while (deferredBlockStreams.size() > 1) {
1836            // In solid compression mode we need to decompress all leading folder'
1837            // streams to get access to an entry. We defer this until really needed
1838            // so that entire blocks can be skipped without wasting time for decompression.
1839            try (final InputStream stream = deferredBlockStreams.remove(0)) {
1840                IOUtils.skip(stream, Long.MAX_VALUE);
1841            }
1842            compressedBytesReadFromCurrentEntry = 0;
1843        }
1844
1845        return deferredBlockStreams.get(0);
1846    }
1847
1848    /**
1849     * Returns an InputStream for reading the contents of the given entry.
1850     *
1851     * <p>For archives using solid compression randomly accessing
1852     * entries will be significantly slower than reading the archive
1853     * sequentially.</p>
1854     *
1855     * @param entry the entry to get the stream for.
1856     * @return a stream to read the entry from.
1857     * @throws IOException if unable to create an input stream from the zipentry
1858     * @since Compress 1.20
1859     */
1860    public InputStream getInputStream(final SevenZArchiveEntry entry) throws IOException {
1861        int entryIndex = -1;
1862        for (int i = 0; i < this.archive.files.length;i++) {
1863            if (entry == this.archive.files[i]) {
1864                entryIndex = i;
1865                break;
1866            }
1867        }
1868
1869        if (entryIndex < 0) {
1870            throw new IllegalArgumentException("Can not find " + entry.getName() + " in " + this.fileName);
1871        }
1872
1873        buildDecodingStream(entryIndex, true);
1874        currentEntryIndex = entryIndex;
1875        currentFolderIndex = archive.streamMap.fileFolderIndex[entryIndex];
1876        return getCurrentStream();
1877    }
1878
1879    /**
1880     * Reads data into an array of bytes.
1881     *
1882     * @param b the array to write data to
1883     * @return the number of bytes read, or -1 if end of input is reached
1884     * @throws IOException
1885     *             if an I/O error has occurred
1886     */
1887    public int read(final byte[] b) throws IOException {
1888        return read(b, 0, b.length);
1889    }
1890
1891    /**
1892     * Reads data into an array of bytes.
1893     *
1894     * @param b the array to write data to
1895     * @param off offset into the buffer to start filling at
1896     * @param len of bytes to read
1897     * @return the number of bytes read, or -1 if end of input is reached
1898     * @throws IOException
1899     *             if an I/O error has occurred
1900     */
1901    public int read(final byte[] b, final int off, final int len) throws IOException {
1902        if (len == 0) {
1903            return 0;
1904        }
1905        final int cnt = getCurrentStream().read(b, off, len);
1906        if (cnt > 0) {
1907            uncompressedBytesReadFromCurrentEntry += cnt;
1908        }
1909        return cnt;
1910    }
1911
1912    /**
1913     * Provides statistics for bytes read from the current entry.
1914     *
1915     * @return statistics for bytes read from the current entry
1916     * @since 1.17
1917     */
1918    public InputStreamStatistics getStatisticsForCurrentEntry() {
1919        return new InputStreamStatistics() {
1920            @Override
1921            public long getCompressedCount() {
1922                return compressedBytesReadFromCurrentEntry;
1923            }
1924            @Override
1925            public long getUncompressedCount() {
1926                return uncompressedBytesReadFromCurrentEntry;
1927            }
1928        };
1929    }
1930
1931    private static long readUint64(final ByteBuffer in) throws IOException {
1932        // long rather than int as it might get shifted beyond the range of an int
1933        final long firstByte = getUnsignedByte(in);
1934        int mask = 0x80;
1935        long value = 0;
1936        for (int i = 0; i < 8; i++) {
1937            if ((firstByte & mask) == 0) {
1938                return value | ((firstByte & (mask - 1)) << (8 * i));
1939            }
1940            final long nextByte = getUnsignedByte(in);
1941            value |= nextByte << (8 * i);
1942            mask >>>= 1;
1943        }
1944        return value;
1945    }
1946
1947    private static char getChar(final ByteBuffer buf) throws IOException {
1948        if (buf.remaining() < 2) {
1949            throw new EOFException();
1950        }
1951        return buf.getChar();
1952    }
1953
1954    private static int getInt(final ByteBuffer buf) throws IOException {
1955        if (buf.remaining() < 4) {
1956            throw new EOFException();
1957        }
1958        return buf.getInt();
1959    }
1960
1961    private static long getLong(final ByteBuffer buf) throws IOException {
1962        if (buf.remaining() < 8) {
1963            throw new EOFException();
1964        }
1965        return buf.getLong();
1966    }
1967
1968    private static void get(final ByteBuffer buf, final byte[] to) throws IOException {
1969        if (buf.remaining() < to.length) {
1970            throw new EOFException();
1971        }
1972        buf.get(to);
1973    }
1974
1975    private static int getUnsignedByte(final ByteBuffer buf) throws IOException {
1976        if (!buf.hasRemaining()) {
1977            throw new EOFException();
1978        }
1979        return buf.get() & 0xff;
1980    }
1981
1982    /**
1983     * Checks if the signature matches what is expected for a 7z file.
1984     *
1985     * @param signature
1986     *            the bytes to check
1987     * @param length
1988     *            the number of bytes to check
1989     * @return true, if this is the signature of a 7z archive.
1990     * @since 1.8
1991     */
1992    public static boolean matches(final byte[] signature, final int length) {
1993        if (length < sevenZSignature.length) {
1994            return false;
1995        }
1996
1997        for (int i = 0; i < sevenZSignature.length; i++) {
1998            if (signature[i] != sevenZSignature[i]) {
1999                return false;
2000            }
2001        }
2002        return true;
2003    }
2004
2005    private static long skipBytesFully(final ByteBuffer input, long bytesToSkip) throws IOException {
2006        if (bytesToSkip < 1) {
2007            return 0;
2008        }
2009        final int current = input.position();
2010        final int maxSkip = input.remaining();
2011        if (maxSkip < bytesToSkip) {
2012            bytesToSkip = maxSkip;
2013        }
2014        input.position(current + (int) bytesToSkip);
2015        return bytesToSkip;
2016    }
2017
2018    private void readFully(final ByteBuffer buf) throws IOException {
2019        buf.rewind();
2020        IOUtils.readFully(channel, buf);
2021        buf.flip();
2022    }
2023
2024    @Override
2025    public String toString() {
2026      return archive.toString();
2027    }
2028
2029    /**
2030     * Derives a default file name from the archive name - if known.
2031     *
2032     * <p>This implements the same heuristics the 7z tools use. In
2033     * 7z's case if an archive contains entries without a name -
2034     * i.e. {@link SevenZArchiveEntry#getName} returns {@code null} -
2035     * then its command line and GUI tools will use this default name
2036     * when extracting the entries.</p>
2037     *
2038     * @return null if the name of the archive is unknown. Otherwise
2039     * if the name of the archive has got any extension, it is
2040     * stripped and the remainder returned. Finally if the name of the
2041     * archive hasn't got any extension then a {@code ~} character is
2042     * appended to the archive name.
2043     *
2044     * @since 1.19
2045     */
2046    public String getDefaultName() {
2047        if (DEFAULT_FILE_NAME.equals(fileName) || fileName == null) {
2048            return null;
2049        }
2050
2051        final String lastSegment = new File(fileName).getName();
2052        final int dotPos = lastSegment.lastIndexOf(".");
2053        if (dotPos > 0) { // if the file starts with a dot then this is not an extension
2054            return lastSegment.substring(0, dotPos);
2055        }
2056        return lastSegment + "~";
2057    }
2058
2059    private static final CharsetEncoder PASSWORD_ENCODER = StandardCharsets.UTF_16LE.newEncoder();
2060
2061    private static byte[] utf16Decode(final char[] chars) throws IOException {
2062        if (chars == null) {
2063            return null;
2064        }
2065        final ByteBuffer encoded = PASSWORD_ENCODER.encode(CharBuffer.wrap(chars));
2066        if (encoded.hasArray()) {
2067            return encoded.array();
2068        }
2069        final byte[] e = new byte[encoded.remaining()];
2070        encoded.get(e);
2071        return e;
2072    }
2073
2074    private static int assertFitsIntoNonNegativeInt(final String what, final long value) throws IOException {
2075        if (value > Integer.MAX_VALUE || value < 0) {
2076            throw new IOException("Cannot handle " + what + " " + value);
2077        }
2078        return (int) value;
2079    }
2080
2081    private static class ArchiveStatistics {
2082        private int numberOfPackedStreams;
2083        private long numberOfCoders;
2084        private long numberOfOutStreams;
2085        private long numberOfInStreams;
2086        private long numberOfUnpackSubStreams;
2087        private int numberOfFolders;
2088        private BitSet folderHasCrc;
2089        private int numberOfEntries;
2090        private int numberOfEntriesWithStream;
2091
2092        @Override
2093        public String toString() {
2094            return "Archive with " + numberOfEntries + " entries in " + numberOfFolders
2095                + " folders. Estimated size " + (estimateSize()/1024l) + " kB.";
2096        }
2097
2098        long estimateSize() {
2099            long lowerBound = 16l * numberOfPackedStreams /* packSizes, packCrcs in Archive */
2100                + numberOfPackedStreams / 8 /* packCrcsDefined in Archive */
2101                + numberOfFolders * folderSize() /* folders in Archive */
2102                + numberOfCoders * coderSize() /* coders in Folder */
2103                + (numberOfOutStreams - numberOfFolders) * bindPairSize() /* bindPairs in Folder */
2104                + 8l * (numberOfInStreams - numberOfOutStreams + numberOfFolders) /* packedStreams in Folder */
2105                + 8l * numberOfOutStreams /* unpackSizes in Folder */
2106                + numberOfEntries * entrySize() /* files in Archive */
2107                + streamMapSize()
2108                ;
2109            return 2 * lowerBound /* conservative guess */;
2110        }
2111
2112        void assertValidity(int maxMemoryLimitInKb) throws IOException {
2113            if (numberOfEntriesWithStream > 0 && numberOfFolders == 0) {
2114                throw new IOException("archive with entries but no folders");
2115            }
2116            if (numberOfEntriesWithStream > numberOfUnpackSubStreams) {
2117                throw new IOException("archive doesn't contain enough substreams for entries");
2118            }
2119
2120            final long memoryNeededInKb = estimateSize() / 1024;
2121            if (maxMemoryLimitInKb < memoryNeededInKb) {
2122                throw new MemoryLimitException(memoryNeededInKb, maxMemoryLimitInKb);
2123            }
2124        }
2125
2126        private long folderSize() {
2127            return 30; /* nested arrays are accounted for separately */
2128        }
2129
2130        private long coderSize() {
2131            return 2 /* methodId is between 1 and four bytes currently, COPY and LZMA2 are the most common with 1 */
2132                + 16
2133                + 4 /* properties, guess */
2134                ;
2135        }
2136
2137        private long bindPairSize() {
2138            return 16;
2139        }
2140
2141        private long entrySize() {
2142            return 100; /* real size depends on name length, everything without name is about 70 bytes */
2143        }
2144
2145        private long streamMapSize() {
2146            return 8 * numberOfFolders /* folderFirstPackStreamIndex, folderFirstFileIndex */
2147                + 8 * numberOfPackedStreams /* packStreamOffsets */
2148                + 4 * numberOfEntries /* fileFolderIndex */
2149                ;
2150        }
2151    }
2152}