View Javadoc
1   /*
2    * Logback: the reliable, generic, fast and flexible logging framework.
3    * Copyright (C) 1999-2026, QOS.ch. All rights reserved.
4    *
5    * This program and the accompanying materials are dual-licensed under
6    * either the terms of the Eclipse Public License v2.0 as published by
7    * the Eclipse Foundation
8    *
9    *   or (per the licensee's choosing)
10   *
11   * under the terms of the GNU Lesser General Public License version 2.1
12   * as published by the Free Software Foundation.
13   */
14  package ch.qos.logback.classic.kryo;
15  
16  import ch.qos.logback.classic.LoggerContext;
17  import com.esotericsoftware.kryo.kryo5.Kryo;
18  import com.esotericsoftware.kryo.kryo5.KryoException;
19  import com.esotericsoftware.kryo.kryo5.Serializer;
20  import com.esotericsoftware.kryo.kryo5.io.Input;
21  import com.esotericsoftware.kryo.kryo5.minlog.Log;
22  
23  import java.io.BufferedInputStream;
24  import java.io.ByteArrayOutputStream;
25  import java.io.EOFException;
26  import java.io.File;
27  import java.io.FileInputStream;
28  import java.io.IOException;
29  import java.util.ArrayList;
30  import java.util.Arrays;
31  import java.util.List;
32  import java.util.function.Consumer;
33  
34  import static ch.qos.logback.classic.kryo.KryoClassicConstants.*;
35  
36  public class KryoDecoder {
37  
38  
39      Kryo kryo;
40      Input input;
41  
42      /** Bytes pushed back after a resync found a HEADER inside a previously consumed buffer. */
43      byte[] unread;
44      int unreadPos;
45  
46      KryoEncoder.SerializerType serializerType;
47      byte streamVersion;
48  
49      /**
50       * Open a Kryo stream and configure the decoder from its header.
51       */
52      public KryoDecoder(File file) throws IOException {
53          this.input = new Input(new BufferedInputStream(new FileInputStream(file)));
54          readAndApplyHeader();
55      }
56  
57      /**
58       * Open a Kryo stream. The header determines version and serializer type;
59       * {@code serializerType} is retained for call-site compatibility and is
60       * checked against the header when present.
61       */
62      public KryoDecoder(KryoEncoder.SerializerType serializerType, File file) throws IOException {
63          this(file);
64          if (this.serializerType != serializerType) {
65              throw new IOException("Stream header serializer type is " + this.serializerType
66                      + " but caller requested " + serializerType);
67          }
68      }
69  
70      void readAndApplyHeader() throws IOException {
71          if (input.available() < STREAM_HEADER_SIZE) {
72              throw new IOException("Stream too short for stream header (need "
73                      + STREAM_HEADER_SIZE + " bytes)");
74          }
75          byte[] rec = input.readBytes(STREAM_HEADER_SIZE);
76          if (!StreamFrameV5.isStreamHeader(rec)) {
77              throw new IOException("Invalid stream magic: expected STWD, got "
78                      + formatBytes(Arrays.copyOf(rec, STREAM_MAGIC.length)));
79          }
80          try {
81              applyHeaderBytes(rec);
82          } catch (IllegalArgumentException e) {
83              throw new IOException(e.getMessage(), e);
84          }
85      }
86  
87      void applyHeaderBytes(byte[] rec) {
88          streamVersion = rec[4];
89          byte flags = rec[5];
90          serializerType = ((flags & FLAG_CUSTOM_SERIALIZER) != 0)
91                  ? KryoEncoder.SerializerType.Custom
92                  : KryoEncoder.SerializerType.Vanilla;
93          Log.debug("Stream version is " + streamVersion);
94          rebuildKryo();
95      }
96  
97      void rebuildKryo() {
98          FieldByteAccumulator saved = currentFieldByteAccumulator();
99          kryo = KryoBuilder.build(serializerType, streamVersion);
100         FieldByteAccumulator next = currentFieldByteAccumulator();
101         if (next != null) {
102             next.merge(saved);
103         }
104     }
105 
106     private FieldByteAccumulator currentFieldByteAccumulator() {
107         if (kryo == null) {
108             return null;
109         }
110         Serializer<?> serializer = kryo.getSerializer(LoggingEventKryoVO.class);
111         if (serializer instanceof FieldByteAccumulatorAware) {
112             return ((FieldByteAccumulatorAware) serializer).getFieldByteAccumulator();
113         }
114         return null;
115     }
116 
117     public KryoEncoder.SerializerType getSerializerType() {
118         return serializerType;
119     }
120 
121     public byte getStreamVersion() {
122         return streamVersion;
123     }
124 
125     private static String formatBytes(byte[] bytes) {
126         StringBuilder sb = new StringBuilder();
127         for (int i = 0; i < bytes.length; i++) {
128             if (i > 0) {
129                 sb.append(' ');
130             }
131             sb.append(String.format("%02X", bytes[i]));
132         }
133         return sb.toString();
134     }
135 
136 
137     /**
138      * Outcome of a decode pass: event count and nanoseconds spent reading objects
139      * versus invoking the consumer, plus what the decoder met on the way.
140      *
141      * @param resyncCount   how many times the decoder met damage (a torn or corrupt frame,
142      *                      an undecodable event) and had to resynchronise or give up
143      * @param endedMidEvent whether the stream ended inside a frame or event
144      * @param bytesConsumed bytes of the headers and frames which were decoded successfully
145      */
146     public static final class DecodeStats {
147 
148         private final int count;
149         private final long decodeNanos;
150         private final long consumeNanos;
151         private final int resyncCount;
152         private final boolean endedMidEvent;
153         private final long bytesConsumed;
154 
155         public DecodeStats(int count, long decodeNanos, long consumeNanos,
156                            int resyncCount, boolean endedMidEvent, long bytesConsumed) {
157             this.count = count;
158             this.decodeNanos = decodeNanos;
159             this.consumeNanos = consumeNanos;
160             this.resyncCount = resyncCount;
161             this.endedMidEvent = endedMidEvent;
162             this.bytesConsumed = bytesConsumed;
163         }
164 
165         public int count() {
166             return count;
167         }
168 
169         public long decodeNanos() {
170             return decodeNanos;
171         }
172 
173         public long consumeNanos() {
174             return consumeNanos;
175         }
176 
177         public int resyncCount() {
178             return resyncCount;
179         }
180 
181         public boolean endedMidEvent() {
182             return endedMidEvent;
183         }
184 
185         public long bytesConsumed() {
186             return bytesConsumed;
187         }
188 
189         /** True if some events may be missing: the file was damaged or cut short. */
190         public boolean truncated() {
191             return resyncCount > 0 || endedMidEvent;
192         }
193 
194         @Override
195         public boolean equals(Object o) {
196             if (this == o)
197                 return true;
198             if (!(o instanceof DecodeStats))
199                 return false;
200             DecodeStats that = (DecodeStats) o;
201             return count == that.count && decodeNanos == that.decodeNanos && consumeNanos == that.consumeNanos
202                     && resyncCount == that.resyncCount && endedMidEvent == that.endedMidEvent
203                     && bytesConsumed == that.bytesConsumed;
204         }
205 
206         @Override
207         public int hashCode() {
208             return java.util.Objects.hash(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
209         }
210 
211         @Override
212         public String toString() {
213             return "DecodeStats[count=" + count + ", decodeNanos=" + decodeNanos + ", consumeNanos=" + consumeNanos
214                     + ", resyncCount=" + resyncCount + ", endedMidEvent=" + endedMidEvent
215                     + ", bytesConsumed=" + bytesConsumed + "]";
216         }
217     }
218 
219     private int resyncCount;
220     private boolean endedMidEvent;
221     private long bytesConsumed;
222 
223     private DecodeStats stats(int count, long decodeNanos, long consumeNanos) {
224         return new DecodeStats(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
225     }
226 
227     public DecodeStats decodeWithConsumer(Consumer<LoggingEventKryoVO> consumer) throws IOException {
228         Log.trace("Decoding Kryo called with consumer");
229         resyncCount = 0;
230         endedMidEvent = false;
231         bytesConsumed = STREAM_HEADER_SIZE;
232         try {
233             if (isFramed(streamVersion)) {
234                 return decodeFramed(consumer);
235             }
236             return decodeUnframed(consumer);
237         } finally {
238             input.close();
239         }
240     }
241 
242     private DecodeStats decodeUnframed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
243         int count = 0;
244         long decodeNanos = 0L;
245         long consumeNanos = 0L;
246         try {
247             while (input.available() > 0) {
248                 long decodeStart = System.nanoTime();
249                 LoggingEventKryoVO event = kryo.readObject(input, LoggingEventKryoVO.class);
250                 long consumeStart = System.nanoTime();
251                 consumer.accept(event);
252                 long consumeEnd = System.nanoTime();
253                 decodeNanos += consumeStart - decodeStart;
254                 consumeNanos += consumeEnd - consumeStart;
255                 count++;
256                 bytesConsumed = input.total();
257             }
258             return stats(count, decodeNanos, consumeNanos);
259         } catch (EOFException | KryoException e) {
260             Log.error("KryoDecoder#decodeUnframed: Failed to decode kryo stream count="+count, e);
261             // an unframed stream cannot be resynchronised: the rest of it is lost
262             resyncCount++;
263             endedMidEvent = true;
264             return stats(count, decodeNanos, consumeNanos);
265         }
266     }
267 
268     private DecodeStats decodeFramed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
269         int count = 0;
270         long decodeNanos = 0L;
271         long consumeNanos = 0L;
272         statsHolder.count = 0;
273         try {
274             while (true) {
275                 int before = count;
276                 boolean more = decodeOneV5(consumer, statsHolder);
277                 if (!more) {
278                     break;
279                 }
280                 if (statsHolder.count > before) {
281                     decodeNanos += statsHolder.decodeNanos;
282                     consumeNanos += statsHolder.consumeNanos;
283                     count = statsHolder.count;
284                 }
285             }
286             return stats(count, decodeNanos, consumeNanos);
287         } catch (KryoException | IllegalArgumentException e) {
288             // IllegalArgumentException: a later header announces an unsupported stream version
289             Log.error("KryoDecoder#decodeFramed: Failed to decode kryo stream count="+count, e);
290             resyncCount++;
291             return stats(count, decodeNanos, consumeNanos);
292         }
293     }
294 
295     private final FrameLoopStats statsHolder = new FrameLoopStats();
296 
297     private static final class FrameLoopStats {
298         int count;
299         long decodeNanos;
300         long consumeNanos;
301     }
302 
303     private boolean decodeOneV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats) {
304         int b0 = readRawByte();
305         if (b0 < 0) {
306             return false;
307         }
308         if (b0 == (STREAM_MAGIC[0] & 0xFF)) {
309             byte[] rec = new byte[STREAM_HEADER_SIZE];
310             rec[0] = (byte) b0;
311             int got = readRaw(rec, 1, STREAM_HEADER_SIZE - 1);
312             if (got < STREAM_HEADER_SIZE - 1) {
313                 endedMidEvent = true;
314                 return false;
315             }
316             if (StreamFrameV5.isStreamHeader(rec)) {
317                 applyHeaderBytes(rec);
318                 bytesConsumed += rec.length;
319                 return true;
320             }
321             return resyncToHeader(rec);
322         }
323         int type = StreamFrameV5.typeOf(b0);
324         ByteArrayOutputStream seed = new ByteArrayOutputStream(8);
325         seed.write(b0);
326         int len = readUnsignedVarInt(seed);
327         if (len < 0) {
328             return resyncToHeader(seed.toByteArray());
329         }
330         byte[] prefix = seed.toByteArray();
331         byte[] lenBytes = java.util.Arrays.copyOfRange(prefix, 1, prefix.length);
332         if (type == StreamFrameV5.TYPE_CLOSE && len == 0) {
333             if (!StreamFrameV5.crcMatches(b0, lenBytes, new byte[0], 0, 0)) {
334                 Log.error("KryoDecoder: V5 CLOSE CRC mismatch");
335                 return resyncToHeader(prefix);
336             }
337             rebuildKryo();
338             bytesConsumed += prefix.length;
339             return true;
340         }
341         if (type != StreamFrameV5.TYPE_EVENT) {
342             Log.error("KryoDecoder: unknown V5 frame type " + type);
343             return resyncToHeader(prefix);
344         }
345         return finishEventV5(consumer, stats, prefix, b0, lenBytes, len);
346     }
347 
348     private boolean finishEventV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
349                                   byte[] prefix, int packedByte, byte[] lenBytes, int len) {
350         if (len > KryoEncoder.MAX_BUFFER_SIZE) {
351             Log.error("KryoDecoder: EVENT length " + len + " exceeds max");
352             return resyncToHeader(prefix);
353         }
354         byte[] payload = new byte[len];
355         int got = readRaw(payload, 0, len);
356         if (got < len) {
357             // the input ran out inside the payload
358             endedMidEvent = true;
359             Log.error("KryoDecoder: truncated EVENT payload count=" + stats.count
360                     + " got=" + got + " expected=" + len);
361             return resyncToHeader(got <= 0 ? prefix : concat(prefix, payload, got));
362         }
363         if (StreamFrameV5.indexOfMagic(payload, 0, payload.length) >= 0) {
364             Log.error("KryoDecoder: STWD inside V5 EVENT payload at count=" + stats.count
365                     + ", treating as torn write");
366             return resyncToHeader(concat(prefix, payload, payload.length));
367         }
368         if (!StreamFrameV5.crcMatches(packedByte, lenBytes, payload, 0, len)) {
369             Log.error("KryoDecoder: EVENT CRC mismatch at count=" + stats.count);
370             if (!resyncToHeader(concat(prefix, payload, payload.length))) {
371                 rebuildKryo();
372             }
373             return true;
374         }
375         return acceptEvent(consumer, stats, payload, concat(prefix, payload, payload.length));
376     }
377 
378     private boolean acceptEvent(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
379                                 byte[] payload, byte[] seedOnFail) {
380         LoggingEventKryoVO event;
381         long decodeStart = System.nanoTime();
382         try {
383             event = kryo.readObject(new Input(payload), LoggingEventKryoVO.class);
384         } catch (RuntimeException e) {
385             Log.error("KryoDecoder: EVENT payload decode failed count=" + stats.count, e);
386             if (!resyncToHeader(seedOnFail)) {
387                 rebuildKryo();
388             }
389             return true;
390         }
391         long consumeStart = System.nanoTime();
392         consumer.accept(event);
393         long consumeEnd = System.nanoTime();
394         stats.decodeNanos = consumeStart - decodeStart;
395         stats.consumeNanos = consumeEnd - consumeStart;
396         stats.count++;
397         bytesConsumed += seedOnFail.length;
398         return true;
399     }
400 
401     /** @return length, or -1 on EOF/overflow */
402     private int readUnsignedVarInt(ByteArrayOutputStream seed) {
403         int result = 0;
404         int shift = 0;
405         while (shift <= 28) {
406             int b = readRawByte();
407             if (b < 0) {
408                 endedMidEvent = true;
409                 return -1;
410             }
411             seed.write(b);
412             result |= (b & 0x7F) << shift;
413             if ((b & 0x80) == 0) {
414                 return result;
415             }
416             shift += 7;
417         }
418         return -1;
419     }
420 
421     /**
422      * Scan {@code seed} then the remaining input for the next {@code STWD} header.
423      * Bytes after that header in {@code seed} are pushed back so the next frame
424      * read sees them.
425      *
426      * @return true if a header was found and applied
427      */
428     boolean resyncToHeader(byte[] seed) {
429         resyncCount++;
430         byte[] window = seed != null ? seed : new byte[0];
431         int idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
432         try {
433             while (idx < 0) {
434                 int b = readRawByte();
435                 if (b < 0) {
436                     return false;
437                 }
438                 window = appendByte(window, (byte) b);
439                 int keep = STREAM_MAGIC.length - 1;
440                 if (window.length > keep) {
441                     window = Arrays.copyOfRange(window, window.length - keep, window.length);
442                 }
443                 idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
444             }
445             int have = window.length - idx;
446             byte[] header = new byte[STREAM_HEADER_SIZE];
447             int copy = Math.min(have, STREAM_HEADER_SIZE);
448             System.arraycopy(window, idx, header, 0, copy);
449             if (have < STREAM_HEADER_SIZE) {
450                 int need = STREAM_HEADER_SIZE - have;
451                 if (readRaw(header, have, need) != need) {
452                     return false;
453                 }
454             } else if (have > STREAM_HEADER_SIZE) {
455                 prependUnread(Arrays.copyOfRange(window, idx + STREAM_HEADER_SIZE, window.length));
456             }
457             if (!StreamFrameV5.isStreamHeader(header)) {
458                 return false;
459             }
460             applyHeaderBytes(header);
461             bytesConsumed += header.length;
462             return true;
463         } catch (KryoException e) {
464             return false;
465         }
466     }
467 
468     /**
469      * Fills {@code dest[off..off+len)} from the unread pushback buffer, then from
470      * {@link Input#read(byte[], int, int)}. Returns the number of bytes actually copied
471      * (less than {@code len} on EOF). Used for EVENT payloads and 6-byte frame headers.
472      * Resync scanning still uses {@link #readRawByte()}.
473      */
474     private int readRaw(byte[] dest, int off, int len) {
475         int n = 0;
476         if (unread != null && unreadPos < unread.length) {
477             int take = Math.min(len, unread.length - unreadPos);
478             System.arraycopy(unread, unreadPos, dest, off, take);
479             unreadPos += take;
480             n += take;
481             if (unreadPos >= unread.length) {
482                 unread = null;
483                 unreadPos = 0;
484             }
485             if (n == len) {
486                 return n;
487             }
488         }
489         try {
490             while (n < len) {
491                 int r = input.read(dest, off + n, len - n);
492                 if (r <= 0) {
493                     break;
494                 }
495                 n += r;
496             }
497         } catch (KryoException e) {
498             // return bytes already copied
499         }
500         return n;
501     }
502 
503     private int readRawByte() {
504         if (unread != null && unreadPos < unread.length) {
505             int b = unread[unreadPos++] & 0xFF;
506             if (unreadPos >= unread.length) {
507                 unread = null;
508                 unreadPos = 0;
509             }
510             return b;
511         }
512         try {
513             return input.readByte() & 0xFF;
514         } catch (KryoException e) {
515             return -1;
516         }
517     }
518 
519     private void prependUnread(byte[] extra) {
520         if (extra == null || extra.length == 0) {
521             return;
522         }
523         if (unread == null || unreadPos >= unread.length) {
524             unread = extra;
525             unreadPos = 0;
526             return;
527         }
528         byte[] left = Arrays.copyOfRange(unread, unreadPos, unread.length);
529         unread = concat(extra, left, left.length);
530         unreadPos = 0;
531     }
532 
533     private static byte[] concat(byte[] header, byte[] payload, int payloadLen) {
534         byte[] out = new byte[header.length + payloadLen];
535         System.arraycopy(header, 0, out, 0, header.length);
536         System.arraycopy(payload, 0, out, header.length, payloadLen);
537         return out;
538     }
539 
540     private static byte[] appendByte(byte[] src, byte b) {
541         byte[] out = Arrays.copyOf(src, src.length + 1);
542         out[src.length] = b;
543         return out;
544     }
545 
546     public List<LoggingEventKryoVO> decodeAsList() throws IOException {
547         List<LoggingEventKryoVO> loggerEventKryoVOList = new ArrayList<>();
548         Log.trace("input.available()=" + input.available());
549         ListConsumer listConsumer = new ListConsumer(loggerEventKryoVOList);
550 
551         DecodeStats stats = decodeWithConsumer(listConsumer);
552         Log.trace("decodeAsList "+stats.count()+" logging events from kryo stream");
553         return loggerEventKryoVOList;
554     }
555 
556 
557     public DecodeStats decodeAndAppend(LoggerContext loggerContext) throws IOException {
558         LoggingConsumer loggingConsumer = new LoggingConsumer(loggerContext);
559         return decodeWithConsumer(loggingConsumer);
560     }
561 
562 }