001/*
002 * Logback: the reliable, generic, fast and flexible logging framework.
003 * Copyright (C) 1999-2026, QOS.ch. All rights reserved.
004 *
005 * This program and the accompanying materials are dual-licensed under
006 * either the terms of the Eclipse Public License v2.0 as published by
007 * the Eclipse Foundation
008 *
009 *   or (per the licensee's choosing)
010 *
011 * under the terms of the GNU Lesser General Public License version 2.1
012 * as published by the Free Software Foundation.
013 */
014package ch.qos.logback.classic.kryo;
015
016import ch.qos.logback.classic.LoggerContext;
017import com.esotericsoftware.kryo.kryo5.Kryo;
018import com.esotericsoftware.kryo.kryo5.KryoException;
019import com.esotericsoftware.kryo.kryo5.Serializer;
020import com.esotericsoftware.kryo.kryo5.io.Input;
021import com.esotericsoftware.kryo.kryo5.minlog.Log;
022
023import java.io.BufferedInputStream;
024import java.io.ByteArrayOutputStream;
025import java.io.EOFException;
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.List;
032import java.util.function.Consumer;
033
034import static ch.qos.logback.classic.kryo.KryoClassicConstants.*;
035
036public class KryoDecoder {
037
038
039    Kryo kryo;
040    Input input;
041
042    /** Bytes pushed back after a resync found a HEADER inside a previously consumed buffer. */
043    byte[] unread;
044    int unreadPos;
045
046    KryoEncoder.SerializerType serializerType;
047    byte streamVersion;
048
049    /**
050     * Open a Kryo stream and configure the decoder from its header.
051     */
052    public KryoDecoder(File file) throws IOException {
053        this.input = new Input(new BufferedInputStream(new FileInputStream(file)));
054        readAndApplyHeader();
055    }
056
057    /**
058     * Open a Kryo stream. The header determines version and serializer type;
059     * {@code serializerType} is retained for call-site compatibility and is
060     * checked against the header when present.
061     */
062    public KryoDecoder(KryoEncoder.SerializerType serializerType, File file) throws IOException {
063        this(file);
064        if (this.serializerType != serializerType) {
065            throw new IOException("Stream header serializer type is " + this.serializerType
066                    + " but caller requested " + serializerType);
067        }
068    }
069
070    void readAndApplyHeader() throws IOException {
071        if (input.available() < STREAM_HEADER_SIZE) {
072            throw new IOException("Stream too short for stream header (need "
073                    + STREAM_HEADER_SIZE + " bytes)");
074        }
075        byte[] rec = input.readBytes(STREAM_HEADER_SIZE);
076        if (!StreamFrameV5.isStreamHeader(rec)) {
077            throw new IOException("Invalid stream magic: expected STWD, got "
078                    + formatBytes(Arrays.copyOf(rec, STREAM_MAGIC.length)));
079        }
080        try {
081            applyHeaderBytes(rec);
082        } catch (IllegalArgumentException e) {
083            throw new IOException(e.getMessage(), e);
084        }
085    }
086
087    void applyHeaderBytes(byte[] rec) {
088        streamVersion = rec[4];
089        byte flags = rec[5];
090        serializerType = ((flags & FLAG_CUSTOM_SERIALIZER) != 0)
091                ? KryoEncoder.SerializerType.Custom
092                : KryoEncoder.SerializerType.Vanilla;
093        Log.debug("Stream version is " + streamVersion);
094        rebuildKryo();
095    }
096
097    void rebuildKryo() {
098        FieldByteAccumulator saved = currentFieldByteAccumulator();
099        kryo = KryoBuilder.build(serializerType, streamVersion);
100        FieldByteAccumulator next = currentFieldByteAccumulator();
101        if (next != null) {
102            next.merge(saved);
103        }
104    }
105
106    private FieldByteAccumulator currentFieldByteAccumulator() {
107        if (kryo == null) {
108            return null;
109        }
110        Serializer<?> serializer = kryo.getSerializer(LoggingEventKryoVO.class);
111        if (serializer instanceof FieldByteAccumulatorAware) {
112            return ((FieldByteAccumulatorAware) serializer).getFieldByteAccumulator();
113        }
114        return null;
115    }
116
117    public KryoEncoder.SerializerType getSerializerType() {
118        return serializerType;
119    }
120
121    public byte getStreamVersion() {
122        return streamVersion;
123    }
124
125    private static String formatBytes(byte[] bytes) {
126        StringBuilder sb = new StringBuilder();
127        for (int i = 0; i < bytes.length; i++) {
128            if (i > 0) {
129                sb.append(' ');
130            }
131            sb.append(String.format("%02X", bytes[i]));
132        }
133        return sb.toString();
134    }
135
136
137    /**
138     * Outcome of a decode pass: event count and nanoseconds spent reading objects
139     * versus invoking the consumer, plus what the decoder met on the way.
140     *
141     * @param resyncCount   how many times the decoder met damage (a torn or corrupt frame,
142     *                      an undecodable event) and had to resynchronise or give up
143     * @param endedMidEvent whether the stream ended inside a frame or event
144     * @param bytesConsumed bytes of the headers and frames which were decoded successfully
145     */
146    public static final class DecodeStats {
147
148        private final int count;
149        private final long decodeNanos;
150        private final long consumeNanos;
151        private final int resyncCount;
152        private final boolean endedMidEvent;
153        private final long bytesConsumed;
154
155        public DecodeStats(int count, long decodeNanos, long consumeNanos,
156                           int resyncCount, boolean endedMidEvent, long bytesConsumed) {
157            this.count = count;
158            this.decodeNanos = decodeNanos;
159            this.consumeNanos = consumeNanos;
160            this.resyncCount = resyncCount;
161            this.endedMidEvent = endedMidEvent;
162            this.bytesConsumed = bytesConsumed;
163        }
164
165        public int count() {
166            return count;
167        }
168
169        public long decodeNanos() {
170            return decodeNanos;
171        }
172
173        public long consumeNanos() {
174            return consumeNanos;
175        }
176
177        public int resyncCount() {
178            return resyncCount;
179        }
180
181        public boolean endedMidEvent() {
182            return endedMidEvent;
183        }
184
185        public long bytesConsumed() {
186            return bytesConsumed;
187        }
188
189        /** True if some events may be missing: the file was damaged or cut short. */
190        public boolean truncated() {
191            return resyncCount > 0 || endedMidEvent;
192        }
193
194        @Override
195        public boolean equals(Object o) {
196            if (this == o)
197                return true;
198            if (!(o instanceof DecodeStats))
199                return false;
200            DecodeStats that = (DecodeStats) o;
201            return count == that.count && decodeNanos == that.decodeNanos && consumeNanos == that.consumeNanos
202                    && resyncCount == that.resyncCount && endedMidEvent == that.endedMidEvent
203                    && bytesConsumed == that.bytesConsumed;
204        }
205
206        @Override
207        public int hashCode() {
208            return java.util.Objects.hash(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
209        }
210
211        @Override
212        public String toString() {
213            return "DecodeStats[count=" + count + ", decodeNanos=" + decodeNanos + ", consumeNanos=" + consumeNanos
214                    + ", resyncCount=" + resyncCount + ", endedMidEvent=" + endedMidEvent
215                    + ", bytesConsumed=" + bytesConsumed + "]";
216        }
217    }
218
219    private int resyncCount;
220    private boolean endedMidEvent;
221    private long bytesConsumed;
222
223    private DecodeStats stats(int count, long decodeNanos, long consumeNanos) {
224        return new DecodeStats(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
225    }
226
227    public DecodeStats decodeWithConsumer(Consumer<LoggingEventKryoVO> consumer) throws IOException {
228        Log.trace("Decoding Kryo called with consumer");
229        resyncCount = 0;
230        endedMidEvent = false;
231        bytesConsumed = STREAM_HEADER_SIZE;
232        try {
233            if (isFramed(streamVersion)) {
234                return decodeFramed(consumer);
235            }
236            return decodeUnframed(consumer);
237        } finally {
238            input.close();
239        }
240    }
241
242    private DecodeStats decodeUnframed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
243        int count = 0;
244        long decodeNanos = 0L;
245        long consumeNanos = 0L;
246        try {
247            while (input.available() > 0) {
248                long decodeStart = System.nanoTime();
249                LoggingEventKryoVO event = kryo.readObject(input, LoggingEventKryoVO.class);
250                long consumeStart = System.nanoTime();
251                consumer.accept(event);
252                long consumeEnd = System.nanoTime();
253                decodeNanos += consumeStart - decodeStart;
254                consumeNanos += consumeEnd - consumeStart;
255                count++;
256                bytesConsumed = input.total();
257            }
258            return stats(count, decodeNanos, consumeNanos);
259        } catch (EOFException | KryoException e) {
260            Log.error("KryoDecoder#decodeUnframed: Failed to decode kryo stream count="+count, e);
261            // an unframed stream cannot be resynchronised: the rest of it is lost
262            resyncCount++;
263            endedMidEvent = true;
264            return stats(count, decodeNanos, consumeNanos);
265        }
266    }
267
268    private DecodeStats decodeFramed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
269        int count = 0;
270        long decodeNanos = 0L;
271        long consumeNanos = 0L;
272        statsHolder.count = 0;
273        try {
274            while (true) {
275                int before = count;
276                boolean more = decodeOneV5(consumer, statsHolder);
277                if (!more) {
278                    break;
279                }
280                if (statsHolder.count > before) {
281                    decodeNanos += statsHolder.decodeNanos;
282                    consumeNanos += statsHolder.consumeNanos;
283                    count = statsHolder.count;
284                }
285            }
286            return stats(count, decodeNanos, consumeNanos);
287        } catch (KryoException | IllegalArgumentException e) {
288            // IllegalArgumentException: a later header announces an unsupported stream version
289            Log.error("KryoDecoder#decodeFramed: Failed to decode kryo stream count="+count, e);
290            resyncCount++;
291            return stats(count, decodeNanos, consumeNanos);
292        }
293    }
294
295    private final FrameLoopStats statsHolder = new FrameLoopStats();
296
297    private static final class FrameLoopStats {
298        int count;
299        long decodeNanos;
300        long consumeNanos;
301    }
302
303    private boolean decodeOneV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats) {
304        int b0 = readRawByte();
305        if (b0 < 0) {
306            return false;
307        }
308        if (b0 == (STREAM_MAGIC[0] & 0xFF)) {
309            byte[] rec = new byte[STREAM_HEADER_SIZE];
310            rec[0] = (byte) b0;
311            int got = readRaw(rec, 1, STREAM_HEADER_SIZE - 1);
312            if (got < STREAM_HEADER_SIZE - 1) {
313                endedMidEvent = true;
314                return false;
315            }
316            if (StreamFrameV5.isStreamHeader(rec)) {
317                applyHeaderBytes(rec);
318                bytesConsumed += rec.length;
319                return true;
320            }
321            return resyncToHeader(rec);
322        }
323        int type = StreamFrameV5.typeOf(b0);
324        ByteArrayOutputStream seed = new ByteArrayOutputStream(8);
325        seed.write(b0);
326        int len = readUnsignedVarInt(seed);
327        if (len < 0) {
328            return resyncToHeader(seed.toByteArray());
329        }
330        byte[] prefix = seed.toByteArray();
331        byte[] lenBytes = java.util.Arrays.copyOfRange(prefix, 1, prefix.length);
332        if (type == StreamFrameV5.TYPE_CLOSE && len == 0) {
333            if (!StreamFrameV5.crcMatches(b0, lenBytes, new byte[0], 0, 0)) {
334                Log.error("KryoDecoder: V5 CLOSE CRC mismatch");
335                return resyncToHeader(prefix);
336            }
337            rebuildKryo();
338            bytesConsumed += prefix.length;
339            return true;
340        }
341        if (type != StreamFrameV5.TYPE_EVENT) {
342            Log.error("KryoDecoder: unknown V5 frame type " + type);
343            return resyncToHeader(prefix);
344        }
345        return finishEventV5(consumer, stats, prefix, b0, lenBytes, len);
346    }
347
348    private boolean finishEventV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
349                                  byte[] prefix, int packedByte, byte[] lenBytes, int len) {
350        if (len > KryoEncoder.MAX_BUFFER_SIZE) {
351            Log.error("KryoDecoder: EVENT length " + len + " exceeds max");
352            return resyncToHeader(prefix);
353        }
354        byte[] payload = new byte[len];
355        int got = readRaw(payload, 0, len);
356        if (got < len) {
357            // the input ran out inside the payload
358            endedMidEvent = true;
359            Log.error("KryoDecoder: truncated EVENT payload count=" + stats.count
360                    + " got=" + got + " expected=" + len);
361            return resyncToHeader(got <= 0 ? prefix : concat(prefix, payload, got));
362        }
363        if (StreamFrameV5.indexOfMagic(payload, 0, payload.length) >= 0) {
364            Log.error("KryoDecoder: STWD inside V5 EVENT payload at count=" + stats.count
365                    + ", treating as torn write");
366            return resyncToHeader(concat(prefix, payload, payload.length));
367        }
368        if (!StreamFrameV5.crcMatches(packedByte, lenBytes, payload, 0, len)) {
369            Log.error("KryoDecoder: EVENT CRC mismatch at count=" + stats.count);
370            if (!resyncToHeader(concat(prefix, payload, payload.length))) {
371                rebuildKryo();
372            }
373            return true;
374        }
375        return acceptEvent(consumer, stats, payload, concat(prefix, payload, payload.length));
376    }
377
378    private boolean acceptEvent(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
379                                byte[] payload, byte[] seedOnFail) {
380        LoggingEventKryoVO event;
381        long decodeStart = System.nanoTime();
382        try {
383            event = kryo.readObject(new Input(payload), LoggingEventKryoVO.class);
384        } catch (RuntimeException e) {
385            Log.error("KryoDecoder: EVENT payload decode failed count=" + stats.count, e);
386            if (!resyncToHeader(seedOnFail)) {
387                rebuildKryo();
388            }
389            return true;
390        }
391        long consumeStart = System.nanoTime();
392        consumer.accept(event);
393        long consumeEnd = System.nanoTime();
394        stats.decodeNanos = consumeStart - decodeStart;
395        stats.consumeNanos = consumeEnd - consumeStart;
396        stats.count++;
397        bytesConsumed += seedOnFail.length;
398        return true;
399    }
400
401    /** @return length, or -1 on EOF/overflow */
402    private int readUnsignedVarInt(ByteArrayOutputStream seed) {
403        int result = 0;
404        int shift = 0;
405        while (shift <= 28) {
406            int b = readRawByte();
407            if (b < 0) {
408                endedMidEvent = true;
409                return -1;
410            }
411            seed.write(b);
412            result |= (b & 0x7F) << shift;
413            if ((b & 0x80) == 0) {
414                return result;
415            }
416            shift += 7;
417        }
418        return -1;
419    }
420
421    /**
422     * Scan {@code seed} then the remaining input for the next {@code STWD} header.
423     * Bytes after that header in {@code seed} are pushed back so the next frame
424     * read sees them.
425     *
426     * @return true if a header was found and applied
427     */
428    boolean resyncToHeader(byte[] seed) {
429        resyncCount++;
430        byte[] window = seed != null ? seed : new byte[0];
431        int idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
432        try {
433            while (idx < 0) {
434                int b = readRawByte();
435                if (b < 0) {
436                    return false;
437                }
438                window = appendByte(window, (byte) b);
439                int keep = STREAM_MAGIC.length - 1;
440                if (window.length > keep) {
441                    window = Arrays.copyOfRange(window, window.length - keep, window.length);
442                }
443                idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
444            }
445            int have = window.length - idx;
446            byte[] header = new byte[STREAM_HEADER_SIZE];
447            int copy = Math.min(have, STREAM_HEADER_SIZE);
448            System.arraycopy(window, idx, header, 0, copy);
449            if (have < STREAM_HEADER_SIZE) {
450                int need = STREAM_HEADER_SIZE - have;
451                if (readRaw(header, have, need) != need) {
452                    return false;
453                }
454            } else if (have > STREAM_HEADER_SIZE) {
455                prependUnread(Arrays.copyOfRange(window, idx + STREAM_HEADER_SIZE, window.length));
456            }
457            if (!StreamFrameV5.isStreamHeader(header)) {
458                return false;
459            }
460            applyHeaderBytes(header);
461            bytesConsumed += header.length;
462            return true;
463        } catch (KryoException e) {
464            return false;
465        }
466    }
467
468    /**
469     * Fills {@code dest[off..off+len)} from the unread pushback buffer, then from
470     * {@link Input#read(byte[], int, int)}. Returns the number of bytes actually copied
471     * (less than {@code len} on EOF). Used for EVENT payloads and 6-byte frame headers.
472     * Resync scanning still uses {@link #readRawByte()}.
473     */
474    private int readRaw(byte[] dest, int off, int len) {
475        int n = 0;
476        if (unread != null && unreadPos < unread.length) {
477            int take = Math.min(len, unread.length - unreadPos);
478            System.arraycopy(unread, unreadPos, dest, off, take);
479            unreadPos += take;
480            n += take;
481            if (unreadPos >= unread.length) {
482                unread = null;
483                unreadPos = 0;
484            }
485            if (n == len) {
486                return n;
487            }
488        }
489        try {
490            while (n < len) {
491                int r = input.read(dest, off + n, len - n);
492                if (r <= 0) {
493                    break;
494                }
495                n += r;
496            }
497        } catch (KryoException e) {
498            // return bytes already copied
499        }
500        return n;
501    }
502
503    private int readRawByte() {
504        if (unread != null && unreadPos < unread.length) {
505            int b = unread[unreadPos++] & 0xFF;
506            if (unreadPos >= unread.length) {
507                unread = null;
508                unreadPos = 0;
509            }
510            return b;
511        }
512        try {
513            return input.readByte() & 0xFF;
514        } catch (KryoException e) {
515            return -1;
516        }
517    }
518
519    private void prependUnread(byte[] extra) {
520        if (extra == null || extra.length == 0) {
521            return;
522        }
523        if (unread == null || unreadPos >= unread.length) {
524            unread = extra;
525            unreadPos = 0;
526            return;
527        }
528        byte[] left = Arrays.copyOfRange(unread, unreadPos, unread.length);
529        unread = concat(extra, left, left.length);
530        unreadPos = 0;
531    }
532
533    private static byte[] concat(byte[] header, byte[] payload, int payloadLen) {
534        byte[] out = new byte[header.length + payloadLen];
535        System.arraycopy(header, 0, out, 0, header.length);
536        System.arraycopy(payload, 0, out, header.length, payloadLen);
537        return out;
538    }
539
540    private static byte[] appendByte(byte[] src, byte b) {
541        byte[] out = Arrays.copyOf(src, src.length + 1);
542        out[src.length] = b;
543        return out;
544    }
545
546    public List<LoggingEventKryoVO> decodeAsList() throws IOException {
547        List<LoggingEventKryoVO> loggerEventKryoVOList = new ArrayList<>();
548        Log.trace("input.available()=" + input.available());
549        ListConsumer listConsumer = new ListConsumer(loggerEventKryoVOList);
550
551        DecodeStats stats = decodeWithConsumer(listConsumer);
552        Log.trace("decodeAsList "+stats.count()+" logging events from kryo stream");
553        return loggerEventKryoVOList;
554    }
555
556
557    public DecodeStats decodeAndAppend(LoggerContext loggerContext) throws IOException {
558        LoggingConsumer loggingConsumer = new LoggingConsumer(loggerContext);
559        return decodeWithConsumer(loggingConsumer);
560    }
561
562}