1
2
3
4
5
6
7
8
9
10
11
12
13
14 package ch.qos.logback.classic.kryo;
15
16 import ch.qos.logback.classic.LoggerContext;
17 import com.esotericsoftware.kryo.kryo5.Kryo;
18 import com.esotericsoftware.kryo.kryo5.KryoException;
19 import com.esotericsoftware.kryo.kryo5.Serializer;
20 import com.esotericsoftware.kryo.kryo5.io.Input;
21 import com.esotericsoftware.kryo.kryo5.minlog.Log;
22
23 import java.io.BufferedInputStream;
24 import java.io.ByteArrayOutputStream;
25 import java.io.EOFException;
26 import java.io.File;
27 import java.io.FileInputStream;
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.List;
32 import java.util.function.Consumer;
33
34 import static ch.qos.logback.classic.kryo.KryoClassicConstants.*;
35
36 public class KryoDecoder {
37
38
39 Kryo kryo;
40 Input input;
41
42
43 byte[] unread;
44 int unreadPos;
45
46 KryoEncoder.SerializerType serializerType;
47 byte streamVersion;
48
49
50
51
52 public KryoDecoder(File file) throws IOException {
53 this.input = new Input(new BufferedInputStream(new FileInputStream(file)));
54 readAndApplyHeader();
55 }
56
57
58
59
60
61
62 public KryoDecoder(KryoEncoder.SerializerType serializerType, File file) throws IOException {
63 this(file);
64 if (this.serializerType != serializerType) {
65 throw new IOException("Stream header serializer type is " + this.serializerType
66 + " but caller requested " + serializerType);
67 }
68 }
69
70 void readAndApplyHeader() throws IOException {
71 if (input.available() < STREAM_HEADER_SIZE) {
72 throw new IOException("Stream too short for stream header (need "
73 + STREAM_HEADER_SIZE + " bytes)");
74 }
75 byte[] rec = input.readBytes(STREAM_HEADER_SIZE);
76 if (!StreamFrameV5.isStreamHeader(rec)) {
77 throw new IOException("Invalid stream magic: expected STWD, got "
78 + formatBytes(Arrays.copyOf(rec, STREAM_MAGIC.length)));
79 }
80 try {
81 applyHeaderBytes(rec);
82 } catch (IllegalArgumentException e) {
83 throw new IOException(e.getMessage(), e);
84 }
85 }
86
87 void applyHeaderBytes(byte[] rec) {
88 streamVersion = rec[4];
89 byte flags = rec[5];
90 serializerType = ((flags & FLAG_CUSTOM_SERIALIZER) != 0)
91 ? KryoEncoder.SerializerType.Custom
92 : KryoEncoder.SerializerType.Vanilla;
93 Log.debug("Stream version is " + streamVersion);
94 rebuildKryo();
95 }
96
97 void rebuildKryo() {
98 FieldByteAccumulator saved = currentFieldByteAccumulator();
99 kryo = KryoBuilder.build(serializerType, streamVersion);
100 FieldByteAccumulator next = currentFieldByteAccumulator();
101 if (next != null) {
102 next.merge(saved);
103 }
104 }
105
106 private FieldByteAccumulator currentFieldByteAccumulator() {
107 if (kryo == null) {
108 return null;
109 }
110 Serializer<?> serializer = kryo.getSerializer(LoggingEventKryoVO.class);
111 if (serializer instanceof FieldByteAccumulatorAware) {
112 return ((FieldByteAccumulatorAware) serializer).getFieldByteAccumulator();
113 }
114 return null;
115 }
116
117 public KryoEncoder.SerializerType getSerializerType() {
118 return serializerType;
119 }
120
121 public byte getStreamVersion() {
122 return streamVersion;
123 }
124
125 private static String formatBytes(byte[] bytes) {
126 StringBuilder sb = new StringBuilder();
127 for (int i = 0; i < bytes.length; i++) {
128 if (i > 0) {
129 sb.append(' ');
130 }
131 sb.append(String.format("%02X", bytes[i]));
132 }
133 return sb.toString();
134 }
135
136
137
138
139
140
141
142
143
144
145
146 public static final class DecodeStats {
147
148 private final int count;
149 private final long decodeNanos;
150 private final long consumeNanos;
151 private final int resyncCount;
152 private final boolean endedMidEvent;
153 private final long bytesConsumed;
154
155 public DecodeStats(int count, long decodeNanos, long consumeNanos,
156 int resyncCount, boolean endedMidEvent, long bytesConsumed) {
157 this.count = count;
158 this.decodeNanos = decodeNanos;
159 this.consumeNanos = consumeNanos;
160 this.resyncCount = resyncCount;
161 this.endedMidEvent = endedMidEvent;
162 this.bytesConsumed = bytesConsumed;
163 }
164
165 public int count() {
166 return count;
167 }
168
169 public long decodeNanos() {
170 return decodeNanos;
171 }
172
173 public long consumeNanos() {
174 return consumeNanos;
175 }
176
177 public int resyncCount() {
178 return resyncCount;
179 }
180
181 public boolean endedMidEvent() {
182 return endedMidEvent;
183 }
184
185 public long bytesConsumed() {
186 return bytesConsumed;
187 }
188
189
190 public boolean truncated() {
191 return resyncCount > 0 || endedMidEvent;
192 }
193
194 @Override
195 public boolean equals(Object o) {
196 if (this == o)
197 return true;
198 if (!(o instanceof DecodeStats))
199 return false;
200 DecodeStats that = (DecodeStats) o;
201 return count == that.count && decodeNanos == that.decodeNanos && consumeNanos == that.consumeNanos
202 && resyncCount == that.resyncCount && endedMidEvent == that.endedMidEvent
203 && bytesConsumed == that.bytesConsumed;
204 }
205
206 @Override
207 public int hashCode() {
208 return java.util.Objects.hash(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
209 }
210
211 @Override
212 public String toString() {
213 return "DecodeStats[count=" + count + ", decodeNanos=" + decodeNanos + ", consumeNanos=" + consumeNanos
214 + ", resyncCount=" + resyncCount + ", endedMidEvent=" + endedMidEvent
215 + ", bytesConsumed=" + bytesConsumed + "]";
216 }
217 }
218
219 private int resyncCount;
220 private boolean endedMidEvent;
221 private long bytesConsumed;
222
223 private DecodeStats stats(int count, long decodeNanos, long consumeNanos) {
224 return new DecodeStats(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed);
225 }
226
227 public DecodeStats decodeWithConsumer(Consumer<LoggingEventKryoVO> consumer) throws IOException {
228 Log.trace("Decoding Kryo called with consumer");
229 resyncCount = 0;
230 endedMidEvent = false;
231 bytesConsumed = STREAM_HEADER_SIZE;
232 try {
233 if (isFramed(streamVersion)) {
234 return decodeFramed(consumer);
235 }
236 return decodeUnframed(consumer);
237 } finally {
238 input.close();
239 }
240 }
241
242 private DecodeStats decodeUnframed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
243 int count = 0;
244 long decodeNanos = 0L;
245 long consumeNanos = 0L;
246 try {
247 while (input.available() > 0) {
248 long decodeStart = System.nanoTime();
249 LoggingEventKryoVO event = kryo.readObject(input, LoggingEventKryoVO.class);
250 long consumeStart = System.nanoTime();
251 consumer.accept(event);
252 long consumeEnd = System.nanoTime();
253 decodeNanos += consumeStart - decodeStart;
254 consumeNanos += consumeEnd - consumeStart;
255 count++;
256 bytesConsumed = input.total();
257 }
258 return stats(count, decodeNanos, consumeNanos);
259 } catch (EOFException | KryoException e) {
260 Log.error("KryoDecoder#decodeUnframed: Failed to decode kryo stream count="+count, e);
261
262 resyncCount++;
263 endedMidEvent = true;
264 return stats(count, decodeNanos, consumeNanos);
265 }
266 }
267
268 private DecodeStats decodeFramed(Consumer<LoggingEventKryoVO> consumer) throws IOException {
269 int count = 0;
270 long decodeNanos = 0L;
271 long consumeNanos = 0L;
272 statsHolder.count = 0;
273 try {
274 while (true) {
275 int before = count;
276 boolean more = decodeOneV5(consumer, statsHolder);
277 if (!more) {
278 break;
279 }
280 if (statsHolder.count > before) {
281 decodeNanos += statsHolder.decodeNanos;
282 consumeNanos += statsHolder.consumeNanos;
283 count = statsHolder.count;
284 }
285 }
286 return stats(count, decodeNanos, consumeNanos);
287 } catch (KryoException | IllegalArgumentException e) {
288
289 Log.error("KryoDecoder#decodeFramed: Failed to decode kryo stream count="+count, e);
290 resyncCount++;
291 return stats(count, decodeNanos, consumeNanos);
292 }
293 }
294
295 private final FrameLoopStats statsHolder = new FrameLoopStats();
296
297 private static final class FrameLoopStats {
298 int count;
299 long decodeNanos;
300 long consumeNanos;
301 }
302
303 private boolean decodeOneV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats) {
304 int b0 = readRawByte();
305 if (b0 < 0) {
306 return false;
307 }
308 if (b0 == (STREAM_MAGIC[0] & 0xFF)) {
309 byte[] rec = new byte[STREAM_HEADER_SIZE];
310 rec[0] = (byte) b0;
311 int got = readRaw(rec, 1, STREAM_HEADER_SIZE - 1);
312 if (got < STREAM_HEADER_SIZE - 1) {
313 endedMidEvent = true;
314 return false;
315 }
316 if (StreamFrameV5.isStreamHeader(rec)) {
317 applyHeaderBytes(rec);
318 bytesConsumed += rec.length;
319 return true;
320 }
321 return resyncToHeader(rec);
322 }
323 int type = StreamFrameV5.typeOf(b0);
324 ByteArrayOutputStream seed = new ByteArrayOutputStream(8);
325 seed.write(b0);
326 int len = readUnsignedVarInt(seed);
327 if (len < 0) {
328 return resyncToHeader(seed.toByteArray());
329 }
330 byte[] prefix = seed.toByteArray();
331 byte[] lenBytes = java.util.Arrays.copyOfRange(prefix, 1, prefix.length);
332 if (type == StreamFrameV5.TYPE_CLOSE && len == 0) {
333 if (!StreamFrameV5.crcMatches(b0, lenBytes, new byte[0], 0, 0)) {
334 Log.error("KryoDecoder: V5 CLOSE CRC mismatch");
335 return resyncToHeader(prefix);
336 }
337 rebuildKryo();
338 bytesConsumed += prefix.length;
339 return true;
340 }
341 if (type != StreamFrameV5.TYPE_EVENT) {
342 Log.error("KryoDecoder: unknown V5 frame type " + type);
343 return resyncToHeader(prefix);
344 }
345 return finishEventV5(consumer, stats, prefix, b0, lenBytes, len);
346 }
347
348 private boolean finishEventV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
349 byte[] prefix, int packedByte, byte[] lenBytes, int len) {
350 if (len > KryoEncoder.MAX_BUFFER_SIZE) {
351 Log.error("KryoDecoder: EVENT length " + len + " exceeds max");
352 return resyncToHeader(prefix);
353 }
354 byte[] payload = new byte[len];
355 int got = readRaw(payload, 0, len);
356 if (got < len) {
357
358 endedMidEvent = true;
359 Log.error("KryoDecoder: truncated EVENT payload count=" + stats.count
360 + " got=" + got + " expected=" + len);
361 return resyncToHeader(got <= 0 ? prefix : concat(prefix, payload, got));
362 }
363 if (StreamFrameV5.indexOfMagic(payload, 0, payload.length) >= 0) {
364 Log.error("KryoDecoder: STWD inside V5 EVENT payload at count=" + stats.count
365 + ", treating as torn write");
366 return resyncToHeader(concat(prefix, payload, payload.length));
367 }
368 if (!StreamFrameV5.crcMatches(packedByte, lenBytes, payload, 0, len)) {
369 Log.error("KryoDecoder: EVENT CRC mismatch at count=" + stats.count);
370 if (!resyncToHeader(concat(prefix, payload, payload.length))) {
371 rebuildKryo();
372 }
373 return true;
374 }
375 return acceptEvent(consumer, stats, payload, concat(prefix, payload, payload.length));
376 }
377
378 private boolean acceptEvent(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats,
379 byte[] payload, byte[] seedOnFail) {
380 LoggingEventKryoVO event;
381 long decodeStart = System.nanoTime();
382 try {
383 event = kryo.readObject(new Input(payload), LoggingEventKryoVO.class);
384 } catch (RuntimeException e) {
385 Log.error("KryoDecoder: EVENT payload decode failed count=" + stats.count, e);
386 if (!resyncToHeader(seedOnFail)) {
387 rebuildKryo();
388 }
389 return true;
390 }
391 long consumeStart = System.nanoTime();
392 consumer.accept(event);
393 long consumeEnd = System.nanoTime();
394 stats.decodeNanos = consumeStart - decodeStart;
395 stats.consumeNanos = consumeEnd - consumeStart;
396 stats.count++;
397 bytesConsumed += seedOnFail.length;
398 return true;
399 }
400
401
402 private int readUnsignedVarInt(ByteArrayOutputStream seed) {
403 int result = 0;
404 int shift = 0;
405 while (shift <= 28) {
406 int b = readRawByte();
407 if (b < 0) {
408 endedMidEvent = true;
409 return -1;
410 }
411 seed.write(b);
412 result |= (b & 0x7F) << shift;
413 if ((b & 0x80) == 0) {
414 return result;
415 }
416 shift += 7;
417 }
418 return -1;
419 }
420
421
422
423
424
425
426
427
428 boolean resyncToHeader(byte[] seed) {
429 resyncCount++;
430 byte[] window = seed != null ? seed : new byte[0];
431 int idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
432 try {
433 while (idx < 0) {
434 int b = readRawByte();
435 if (b < 0) {
436 return false;
437 }
438 window = appendByte(window, (byte) b);
439 int keep = STREAM_MAGIC.length - 1;
440 if (window.length > keep) {
441 window = Arrays.copyOfRange(window, window.length - keep, window.length);
442 }
443 idx = StreamFrameV5.indexOfMagic(window, 0, window.length);
444 }
445 int have = window.length - idx;
446 byte[] header = new byte[STREAM_HEADER_SIZE];
447 int copy = Math.min(have, STREAM_HEADER_SIZE);
448 System.arraycopy(window, idx, header, 0, copy);
449 if (have < STREAM_HEADER_SIZE) {
450 int need = STREAM_HEADER_SIZE - have;
451 if (readRaw(header, have, need) != need) {
452 return false;
453 }
454 } else if (have > STREAM_HEADER_SIZE) {
455 prependUnread(Arrays.copyOfRange(window, idx + STREAM_HEADER_SIZE, window.length));
456 }
457 if (!StreamFrameV5.isStreamHeader(header)) {
458 return false;
459 }
460 applyHeaderBytes(header);
461 bytesConsumed += header.length;
462 return true;
463 } catch (KryoException e) {
464 return false;
465 }
466 }
467
468
469
470
471
472
473
474 private int readRaw(byte[] dest, int off, int len) {
475 int n = 0;
476 if (unread != null && unreadPos < unread.length) {
477 int take = Math.min(len, unread.length - unreadPos);
478 System.arraycopy(unread, unreadPos, dest, off, take);
479 unreadPos += take;
480 n += take;
481 if (unreadPos >= unread.length) {
482 unread = null;
483 unreadPos = 0;
484 }
485 if (n == len) {
486 return n;
487 }
488 }
489 try {
490 while (n < len) {
491 int r = input.read(dest, off + n, len - n);
492 if (r <= 0) {
493 break;
494 }
495 n += r;
496 }
497 } catch (KryoException e) {
498
499 }
500 return n;
501 }
502
503 private int readRawByte() {
504 if (unread != null && unreadPos < unread.length) {
505 int b = unread[unreadPos++] & 0xFF;
506 if (unreadPos >= unread.length) {
507 unread = null;
508 unreadPos = 0;
509 }
510 return b;
511 }
512 try {
513 return input.readByte() & 0xFF;
514 } catch (KryoException e) {
515 return -1;
516 }
517 }
518
519 private void prependUnread(byte[] extra) {
520 if (extra == null || extra.length == 0) {
521 return;
522 }
523 if (unread == null || unreadPos >= unread.length) {
524 unread = extra;
525 unreadPos = 0;
526 return;
527 }
528 byte[] left = Arrays.copyOfRange(unread, unreadPos, unread.length);
529 unread = concat(extra, left, left.length);
530 unreadPos = 0;
531 }
532
533 private static byte[] concat(byte[] header, byte[] payload, int payloadLen) {
534 byte[] out = new byte[header.length + payloadLen];
535 System.arraycopy(header, 0, out, 0, header.length);
536 System.arraycopy(payload, 0, out, header.length, payloadLen);
537 return out;
538 }
539
540 private static byte[] appendByte(byte[] src, byte b) {
541 byte[] out = Arrays.copyOf(src, src.length + 1);
542 out[src.length] = b;
543 return out;
544 }
545
546 public List<LoggingEventKryoVO> decodeAsList() throws IOException {
547 List<LoggingEventKryoVO> loggerEventKryoVOList = new ArrayList<>();
548 Log.trace("input.available()=" + input.available());
549 ListConsumer listConsumer = new ListConsumer(loggerEventKryoVOList);
550
551 DecodeStats stats = decodeWithConsumer(listConsumer);
552 Log.trace("decodeAsList "+stats.count()+" logging events from kryo stream");
553 return loggerEventKryoVOList;
554 }
555
556
557 public DecodeStats decodeAndAppend(LoggerContext loggerContext) throws IOException {
558 LoggingConsumer loggingConsumer = new LoggingConsumer(loggerContext);
559 return decodeWithConsumer(loggingConsumer);
560 }
561
562 }