001/* 002 * Logback: the reliable, generic, fast and flexible logging framework. 003 * Copyright (C) 1999-2026, QOS.ch. All rights reserved. 004 * 005 * This program and the accompanying materials are dual-licensed under 006 * either the terms of the Eclipse Public License v2.0 as published by 007 * the Eclipse Foundation 008 * 009 * or (per the licensee's choosing) 010 * 011 * under the terms of the GNU Lesser General Public License version 2.1 012 * as published by the Free Software Foundation. 013 */ 014package ch.qos.logback.classic.kryo; 015 016import ch.qos.logback.classic.LoggerContext; 017import com.esotericsoftware.kryo.kryo5.Kryo; 018import com.esotericsoftware.kryo.kryo5.KryoException; 019import com.esotericsoftware.kryo.kryo5.Serializer; 020import com.esotericsoftware.kryo.kryo5.io.Input; 021import com.esotericsoftware.kryo.kryo5.minlog.Log; 022 023import java.io.BufferedInputStream; 024import java.io.ByteArrayOutputStream; 025import java.io.EOFException; 026import java.io.File; 027import java.io.FileInputStream; 028import java.io.IOException; 029import java.util.ArrayList; 030import java.util.Arrays; 031import java.util.List; 032import java.util.function.Consumer; 033 034import static ch.qos.logback.classic.kryo.KryoClassicConstants.*; 035 036public class KryoDecoder { 037 038 039 Kryo kryo; 040 Input input; 041 042 /** Bytes pushed back after a resync found a HEADER inside a previously consumed buffer. */ 043 byte[] unread; 044 int unreadPos; 045 046 KryoEncoder.SerializerType serializerType; 047 byte streamVersion; 048 049 /** 050 * Open a Kryo stream and configure the decoder from its header. 051 */ 052 public KryoDecoder(File file) throws IOException { 053 this.input = new Input(new BufferedInputStream(new FileInputStream(file))); 054 readAndApplyHeader(); 055 } 056 057 /** 058 * Open a Kryo stream. The header determines version and serializer type; 059 * {@code serializerType} is retained for call-site compatibility and is 060 * checked against the header when present. 061 */ 062 public KryoDecoder(KryoEncoder.SerializerType serializerType, File file) throws IOException { 063 this(file); 064 if (this.serializerType != serializerType) { 065 throw new IOException("Stream header serializer type is " + this.serializerType 066 + " but caller requested " + serializerType); 067 } 068 } 069 070 void readAndApplyHeader() throws IOException { 071 if (input.available() < STREAM_HEADER_SIZE) { 072 throw new IOException("Stream too short for stream header (need " 073 + STREAM_HEADER_SIZE + " bytes)"); 074 } 075 byte[] rec = input.readBytes(STREAM_HEADER_SIZE); 076 if (!StreamFrameV5.isStreamHeader(rec)) { 077 throw new IOException("Invalid stream magic: expected STWD, got " 078 + formatBytes(Arrays.copyOf(rec, STREAM_MAGIC.length))); 079 } 080 try { 081 applyHeaderBytes(rec); 082 } catch (IllegalArgumentException e) { 083 throw new IOException(e.getMessage(), e); 084 } 085 } 086 087 void applyHeaderBytes(byte[] rec) { 088 streamVersion = rec[4]; 089 byte flags = rec[5]; 090 serializerType = ((flags & FLAG_CUSTOM_SERIALIZER) != 0) 091 ? KryoEncoder.SerializerType.Custom 092 : KryoEncoder.SerializerType.Vanilla; 093 Log.debug("Stream version is " + streamVersion); 094 rebuildKryo(); 095 } 096 097 void rebuildKryo() { 098 FieldByteAccumulator saved = currentFieldByteAccumulator(); 099 kryo = KryoBuilder.build(serializerType, streamVersion); 100 FieldByteAccumulator next = currentFieldByteAccumulator(); 101 if (next != null) { 102 next.merge(saved); 103 } 104 } 105 106 private FieldByteAccumulator currentFieldByteAccumulator() { 107 if (kryo == null) { 108 return null; 109 } 110 Serializer<?> serializer = kryo.getSerializer(LoggingEventKryoVO.class); 111 if (serializer instanceof FieldByteAccumulatorAware) { 112 return ((FieldByteAccumulatorAware) serializer).getFieldByteAccumulator(); 113 } 114 return null; 115 } 116 117 public KryoEncoder.SerializerType getSerializerType() { 118 return serializerType; 119 } 120 121 public byte getStreamVersion() { 122 return streamVersion; 123 } 124 125 private static String formatBytes(byte[] bytes) { 126 StringBuilder sb = new StringBuilder(); 127 for (int i = 0; i < bytes.length; i++) { 128 if (i > 0) { 129 sb.append(' '); 130 } 131 sb.append(String.format("%02X", bytes[i])); 132 } 133 return sb.toString(); 134 } 135 136 137 /** 138 * Outcome of a decode pass: event count and nanoseconds spent reading objects 139 * versus invoking the consumer, plus what the decoder met on the way. 140 * 141 * @param resyncCount how many times the decoder met damage (a torn or corrupt frame, 142 * an undecodable event) and had to resynchronise or give up 143 * @param endedMidEvent whether the stream ended inside a frame or event 144 * @param bytesConsumed bytes of the headers and frames which were decoded successfully 145 */ 146 public static final class DecodeStats { 147 148 private final int count; 149 private final long decodeNanos; 150 private final long consumeNanos; 151 private final int resyncCount; 152 private final boolean endedMidEvent; 153 private final long bytesConsumed; 154 155 public DecodeStats(int count, long decodeNanos, long consumeNanos, 156 int resyncCount, boolean endedMidEvent, long bytesConsumed) { 157 this.count = count; 158 this.decodeNanos = decodeNanos; 159 this.consumeNanos = consumeNanos; 160 this.resyncCount = resyncCount; 161 this.endedMidEvent = endedMidEvent; 162 this.bytesConsumed = bytesConsumed; 163 } 164 165 public int count() { 166 return count; 167 } 168 169 public long decodeNanos() { 170 return decodeNanos; 171 } 172 173 public long consumeNanos() { 174 return consumeNanos; 175 } 176 177 public int resyncCount() { 178 return resyncCount; 179 } 180 181 public boolean endedMidEvent() { 182 return endedMidEvent; 183 } 184 185 public long bytesConsumed() { 186 return bytesConsumed; 187 } 188 189 /** True if some events may be missing: the file was damaged or cut short. */ 190 public boolean truncated() { 191 return resyncCount > 0 || endedMidEvent; 192 } 193 194 @Override 195 public boolean equals(Object o) { 196 if (this == o) 197 return true; 198 if (!(o instanceof DecodeStats)) 199 return false; 200 DecodeStats that = (DecodeStats) o; 201 return count == that.count && decodeNanos == that.decodeNanos && consumeNanos == that.consumeNanos 202 && resyncCount == that.resyncCount && endedMidEvent == that.endedMidEvent 203 && bytesConsumed == that.bytesConsumed; 204 } 205 206 @Override 207 public int hashCode() { 208 return java.util.Objects.hash(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed); 209 } 210 211 @Override 212 public String toString() { 213 return "DecodeStats[count=" + count + ", decodeNanos=" + decodeNanos + ", consumeNanos=" + consumeNanos 214 + ", resyncCount=" + resyncCount + ", endedMidEvent=" + endedMidEvent 215 + ", bytesConsumed=" + bytesConsumed + "]"; 216 } 217 } 218 219 private int resyncCount; 220 private boolean endedMidEvent; 221 private long bytesConsumed; 222 223 private DecodeStats stats(int count, long decodeNanos, long consumeNanos) { 224 return new DecodeStats(count, decodeNanos, consumeNanos, resyncCount, endedMidEvent, bytesConsumed); 225 } 226 227 public DecodeStats decodeWithConsumer(Consumer<LoggingEventKryoVO> consumer) throws IOException { 228 Log.trace("Decoding Kryo called with consumer"); 229 resyncCount = 0; 230 endedMidEvent = false; 231 bytesConsumed = STREAM_HEADER_SIZE; 232 try { 233 if (isFramed(streamVersion)) { 234 return decodeFramed(consumer); 235 } 236 return decodeUnframed(consumer); 237 } finally { 238 input.close(); 239 } 240 } 241 242 private DecodeStats decodeUnframed(Consumer<LoggingEventKryoVO> consumer) throws IOException { 243 int count = 0; 244 long decodeNanos = 0L; 245 long consumeNanos = 0L; 246 try { 247 while (input.available() > 0) { 248 long decodeStart = System.nanoTime(); 249 LoggingEventKryoVO event = kryo.readObject(input, LoggingEventKryoVO.class); 250 long consumeStart = System.nanoTime(); 251 consumer.accept(event); 252 long consumeEnd = System.nanoTime(); 253 decodeNanos += consumeStart - decodeStart; 254 consumeNanos += consumeEnd - consumeStart; 255 count++; 256 bytesConsumed = input.total(); 257 } 258 return stats(count, decodeNanos, consumeNanos); 259 } catch (EOFException | KryoException e) { 260 Log.error("KryoDecoder#decodeUnframed: Failed to decode kryo stream count="+count, e); 261 // an unframed stream cannot be resynchronised: the rest of it is lost 262 resyncCount++; 263 endedMidEvent = true; 264 return stats(count, decodeNanos, consumeNanos); 265 } 266 } 267 268 private DecodeStats decodeFramed(Consumer<LoggingEventKryoVO> consumer) throws IOException { 269 int count = 0; 270 long decodeNanos = 0L; 271 long consumeNanos = 0L; 272 statsHolder.count = 0; 273 try { 274 while (true) { 275 int before = count; 276 boolean more = decodeOneV5(consumer, statsHolder); 277 if (!more) { 278 break; 279 } 280 if (statsHolder.count > before) { 281 decodeNanos += statsHolder.decodeNanos; 282 consumeNanos += statsHolder.consumeNanos; 283 count = statsHolder.count; 284 } 285 } 286 return stats(count, decodeNanos, consumeNanos); 287 } catch (KryoException | IllegalArgumentException e) { 288 // IllegalArgumentException: a later header announces an unsupported stream version 289 Log.error("KryoDecoder#decodeFramed: Failed to decode kryo stream count="+count, e); 290 resyncCount++; 291 return stats(count, decodeNanos, consumeNanos); 292 } 293 } 294 295 private final FrameLoopStats statsHolder = new FrameLoopStats(); 296 297 private static final class FrameLoopStats { 298 int count; 299 long decodeNanos; 300 long consumeNanos; 301 } 302 303 private boolean decodeOneV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats) { 304 int b0 = readRawByte(); 305 if (b0 < 0) { 306 return false; 307 } 308 if (b0 == (STREAM_MAGIC[0] & 0xFF)) { 309 byte[] rec = new byte[STREAM_HEADER_SIZE]; 310 rec[0] = (byte) b0; 311 int got = readRaw(rec, 1, STREAM_HEADER_SIZE - 1); 312 if (got < STREAM_HEADER_SIZE - 1) { 313 endedMidEvent = true; 314 return false; 315 } 316 if (StreamFrameV5.isStreamHeader(rec)) { 317 applyHeaderBytes(rec); 318 bytesConsumed += rec.length; 319 return true; 320 } 321 return resyncToHeader(rec); 322 } 323 int type = StreamFrameV5.typeOf(b0); 324 ByteArrayOutputStream seed = new ByteArrayOutputStream(8); 325 seed.write(b0); 326 int len = readUnsignedVarInt(seed); 327 if (len < 0) { 328 return resyncToHeader(seed.toByteArray()); 329 } 330 byte[] prefix = seed.toByteArray(); 331 byte[] lenBytes = java.util.Arrays.copyOfRange(prefix, 1, prefix.length); 332 if (type == StreamFrameV5.TYPE_CLOSE && len == 0) { 333 if (!StreamFrameV5.crcMatches(b0, lenBytes, new byte[0], 0, 0)) { 334 Log.error("KryoDecoder: V5 CLOSE CRC mismatch"); 335 return resyncToHeader(prefix); 336 } 337 rebuildKryo(); 338 bytesConsumed += prefix.length; 339 return true; 340 } 341 if (type != StreamFrameV5.TYPE_EVENT) { 342 Log.error("KryoDecoder: unknown V5 frame type " + type); 343 return resyncToHeader(prefix); 344 } 345 return finishEventV5(consumer, stats, prefix, b0, lenBytes, len); 346 } 347 348 private boolean finishEventV5(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats, 349 byte[] prefix, int packedByte, byte[] lenBytes, int len) { 350 if (len > KryoEncoder.MAX_BUFFER_SIZE) { 351 Log.error("KryoDecoder: EVENT length " + len + " exceeds max"); 352 return resyncToHeader(prefix); 353 } 354 byte[] payload = new byte[len]; 355 int got = readRaw(payload, 0, len); 356 if (got < len) { 357 // the input ran out inside the payload 358 endedMidEvent = true; 359 Log.error("KryoDecoder: truncated EVENT payload count=" + stats.count 360 + " got=" + got + " expected=" + len); 361 return resyncToHeader(got <= 0 ? prefix : concat(prefix, payload, got)); 362 } 363 if (StreamFrameV5.indexOfMagic(payload, 0, payload.length) >= 0) { 364 Log.error("KryoDecoder: STWD inside V5 EVENT payload at count=" + stats.count 365 + ", treating as torn write"); 366 return resyncToHeader(concat(prefix, payload, payload.length)); 367 } 368 if (!StreamFrameV5.crcMatches(packedByte, lenBytes, payload, 0, len)) { 369 Log.error("KryoDecoder: EVENT CRC mismatch at count=" + stats.count); 370 if (!resyncToHeader(concat(prefix, payload, payload.length))) { 371 rebuildKryo(); 372 } 373 return true; 374 } 375 return acceptEvent(consumer, stats, payload, concat(prefix, payload, payload.length)); 376 } 377 378 private boolean acceptEvent(Consumer<LoggingEventKryoVO> consumer, FrameLoopStats stats, 379 byte[] payload, byte[] seedOnFail) { 380 LoggingEventKryoVO event; 381 long decodeStart = System.nanoTime(); 382 try { 383 event = kryo.readObject(new Input(payload), LoggingEventKryoVO.class); 384 } catch (RuntimeException e) { 385 Log.error("KryoDecoder: EVENT payload decode failed count=" + stats.count, e); 386 if (!resyncToHeader(seedOnFail)) { 387 rebuildKryo(); 388 } 389 return true; 390 } 391 long consumeStart = System.nanoTime(); 392 consumer.accept(event); 393 long consumeEnd = System.nanoTime(); 394 stats.decodeNanos = consumeStart - decodeStart; 395 stats.consumeNanos = consumeEnd - consumeStart; 396 stats.count++; 397 bytesConsumed += seedOnFail.length; 398 return true; 399 } 400 401 /** @return length, or -1 on EOF/overflow */ 402 private int readUnsignedVarInt(ByteArrayOutputStream seed) { 403 int result = 0; 404 int shift = 0; 405 while (shift <= 28) { 406 int b = readRawByte(); 407 if (b < 0) { 408 endedMidEvent = true; 409 return -1; 410 } 411 seed.write(b); 412 result |= (b & 0x7F) << shift; 413 if ((b & 0x80) == 0) { 414 return result; 415 } 416 shift += 7; 417 } 418 return -1; 419 } 420 421 /** 422 * Scan {@code seed} then the remaining input for the next {@code STWD} header. 423 * Bytes after that header in {@code seed} are pushed back so the next frame 424 * read sees them. 425 * 426 * @return true if a header was found and applied 427 */ 428 boolean resyncToHeader(byte[] seed) { 429 resyncCount++; 430 byte[] window = seed != null ? seed : new byte[0]; 431 int idx = StreamFrameV5.indexOfMagic(window, 0, window.length); 432 try { 433 while (idx < 0) { 434 int b = readRawByte(); 435 if (b < 0) { 436 return false; 437 } 438 window = appendByte(window, (byte) b); 439 int keep = STREAM_MAGIC.length - 1; 440 if (window.length > keep) { 441 window = Arrays.copyOfRange(window, window.length - keep, window.length); 442 } 443 idx = StreamFrameV5.indexOfMagic(window, 0, window.length); 444 } 445 int have = window.length - idx; 446 byte[] header = new byte[STREAM_HEADER_SIZE]; 447 int copy = Math.min(have, STREAM_HEADER_SIZE); 448 System.arraycopy(window, idx, header, 0, copy); 449 if (have < STREAM_HEADER_SIZE) { 450 int need = STREAM_HEADER_SIZE - have; 451 if (readRaw(header, have, need) != need) { 452 return false; 453 } 454 } else if (have > STREAM_HEADER_SIZE) { 455 prependUnread(Arrays.copyOfRange(window, idx + STREAM_HEADER_SIZE, window.length)); 456 } 457 if (!StreamFrameV5.isStreamHeader(header)) { 458 return false; 459 } 460 applyHeaderBytes(header); 461 bytesConsumed += header.length; 462 return true; 463 } catch (KryoException e) { 464 return false; 465 } 466 } 467 468 /** 469 * Fills {@code dest[off..off+len)} from the unread pushback buffer, then from 470 * {@link Input#read(byte[], int, int)}. Returns the number of bytes actually copied 471 * (less than {@code len} on EOF). Used for EVENT payloads and 6-byte frame headers. 472 * Resync scanning still uses {@link #readRawByte()}. 473 */ 474 private int readRaw(byte[] dest, int off, int len) { 475 int n = 0; 476 if (unread != null && unreadPos < unread.length) { 477 int take = Math.min(len, unread.length - unreadPos); 478 System.arraycopy(unread, unreadPos, dest, off, take); 479 unreadPos += take; 480 n += take; 481 if (unreadPos >= unread.length) { 482 unread = null; 483 unreadPos = 0; 484 } 485 if (n == len) { 486 return n; 487 } 488 } 489 try { 490 while (n < len) { 491 int r = input.read(dest, off + n, len - n); 492 if (r <= 0) { 493 break; 494 } 495 n += r; 496 } 497 } catch (KryoException e) { 498 // return bytes already copied 499 } 500 return n; 501 } 502 503 private int readRawByte() { 504 if (unread != null && unreadPos < unread.length) { 505 int b = unread[unreadPos++] & 0xFF; 506 if (unreadPos >= unread.length) { 507 unread = null; 508 unreadPos = 0; 509 } 510 return b; 511 } 512 try { 513 return input.readByte() & 0xFF; 514 } catch (KryoException e) { 515 return -1; 516 } 517 } 518 519 private void prependUnread(byte[] extra) { 520 if (extra == null || extra.length == 0) { 521 return; 522 } 523 if (unread == null || unreadPos >= unread.length) { 524 unread = extra; 525 unreadPos = 0; 526 return; 527 } 528 byte[] left = Arrays.copyOfRange(unread, unreadPos, unread.length); 529 unread = concat(extra, left, left.length); 530 unreadPos = 0; 531 } 532 533 private static byte[] concat(byte[] header, byte[] payload, int payloadLen) { 534 byte[] out = new byte[header.length + payloadLen]; 535 System.arraycopy(header, 0, out, 0, header.length); 536 System.arraycopy(payload, 0, out, header.length, payloadLen); 537 return out; 538 } 539 540 private static byte[] appendByte(byte[] src, byte b) { 541 byte[] out = Arrays.copyOf(src, src.length + 1); 542 out[src.length] = b; 543 return out; 544 } 545 546 public List<LoggingEventKryoVO> decodeAsList() throws IOException { 547 List<LoggingEventKryoVO> loggerEventKryoVOList = new ArrayList<>(); 548 Log.trace("input.available()=" + input.available()); 549 ListConsumer listConsumer = new ListConsumer(loggerEventKryoVOList); 550 551 DecodeStats stats = decodeWithConsumer(listConsumer); 552 Log.trace("decodeAsList "+stats.count()+" logging events from kryo stream"); 553 return loggerEventKryoVOList; 554 } 555 556 557 public DecodeStats decodeAndAppend(LoggerContext loggerContext) throws IOException { 558 LoggingConsumer loggingConsumer = new LoggingConsumer(loggerContext); 559 return decodeWithConsumer(loggingConsumer); 560 } 561 562}