聊聊flink的AscendingTimestampExtractor

序本文主要研究一下flink的AscendingTimestampExtractorAscendingTimestampExtractorflink-streaming-java_2.11-1.7.0-sources.jar!/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java/** * A timestamp assigner and watermark generator for streams where timestamps are monotonously * ascending. In this case, the local watermarks for the streams are easy to generate, because * they strictly follow the timestamps. * * @param <T> The type of the elements that this function can extract timestamps from /@PublicEvolvingpublic abstract class AscendingTimestampExtractor<T> implements AssignerWithPeriodicWatermarks<T> { private static final long serialVersionUID = 1L; /* The current timestamp. / private long currentTimestamp = Long.MIN_VALUE; /* Handler that is called when timestamp monotony is violated. / private MonotonyViolationHandler violationHandler = new LoggingHandler(); /* * Extracts the timestamp from the given element. The timestamp must be monotonically increasing. * * @param element The element that the timestamp is extracted from. * @return The new timestamp. / public abstract long extractAscendingTimestamp(T element); /* * Sets the handler for violations to the ascending timestamp order. * * @param handler The violation handler to use. * @return This extractor. / public AscendingTimestampExtractor<T> withViolationHandler(MonotonyViolationHandler handler) { this.violationHandler = requireNonNull(handler); return this; } // ———————————————————————— @Override public final long extractTimestamp(T element, long elementPrevTimestamp) { final long newTimestamp = extractAscendingTimestamp(element); if (newTimestamp >= this.currentTimestamp) { this.currentTimestamp = newTimestamp; return newTimestamp; } else { violationHandler.handleViolation(newTimestamp, this.currentTimestamp); return newTimestamp; } } @Override public final Watermark getCurrentWatermark() { return new Watermark(currentTimestamp == Long.MIN_VALUE ? Long.MIN_VALUE : currentTimestamp - 1); } //……}AscendingTimestampExtractor抽象类实现AssignerWithPeriodicWatermarks接口的extractTimestamp及getCurrentWatermark方法，同时声明抽象方法extractAscendingTimestamp供子类实现AscendingTimestampExtractor适用于elements的时间在每个parallel task里头是单调递增(timestamp monotony)的场景，extractTimestamp这里先是调用子类实现的extractAscendingTimestamp方法从element提取newTimestamp，然后返回，对于违反timestamp monotony的，这里调用MonotonyViolationHandler进行处理getCurrentWatermark方法在currentTimestamp不为Long.MIN_VALUE时返回Watermark(currentTimestamp - 1)MonotonyViolationHandlerflink-streaming-java_2.11-1.7.0-sources.jar!/org/apache/flink/streaming/api/functions/timestamps/AscendingTimestampExtractor.java /* * Interface for handlers that handle violations of the monotonous ascending timestamps * property. / public interface MonotonyViolationHandler extends java.io.Serializable { /* * Called when the property of monotonously ascending timestamps is violated, i.e., * when {@code elementTimestamp < lastTimestamp}. * * @param elementTimestamp The timestamp of the current element. * @param lastTimestamp The last timestamp. / void handleViolation(long elementTimestamp, long lastTimestamp); } /* * Handler that does nothing when timestamp monotony is violated. / public static final class IgnoringHandler implements MonotonyViolationHandler { private static final long serialVersionUID = 1L; @Override public void handleViolation(long elementTimestamp, long lastTimestamp) {} } /* * Handler that fails the program when timestamp monotony is violated. / public static final class FailingHandler implements MonotonyViolationHandler { private static final long serialVersionUID = 1L; @Override public void handleViolation(long elementTimestamp, long lastTimestamp) { throw new RuntimeException(“Ascending timestamps condition violated. Element timestamp " + elementTimestamp + " is smaller than last timestamp " + lastTimestamp); } } /* * Handler that only logs violations of timestamp monotony, on WARN log level. */ public static final class LoggingHandler implements MonotonyViolationHandler { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(AscendingTimestampExtractor.class); @Override public void handleViolation(long elementTimestamp, long lastTimestamp) { LOG.warn(“Timestamp monotony violated: {} < {}”, elementTimestamp, lastTimestamp); } }MonotonyViolationHandler继承了Serializable，它定义了handleViolation方法，这个接口内置有三个实现类，分别是IgnoringHandler、FailingHandler、FailingHandlerIgnoringHandler的handleViolation方法不做任何处理；FailingHandler的handleViolation会抛出RuntimeException；LoggingHandler的handleViolation方法会打印warn日志AscendingTimestampExtractor默认使用的是LoggingHandler，也可以通过withViolationHandler方法来进行设置实例 @Test public void testWithFailingHandler() { AscendingTimestampExtractor<Long> extractor = (new AscendingTimestampExtractorTest.LongExtractor()).withViolationHandler(new FailingHandler()); this.runValidTests(extractor); try { this.runInvalidTest(extractor); Assert.fail(“should fail with an exception”); } catch (Exception var3) { ; } } private void runValidTests(AscendingTimestampExtractor<Long> extractor) { Assert.assertEquals(13L, extractor.extractTimestamp(13L, -1L)); Assert.assertEquals(13L, extractor.extractTimestamp(13L, 0L)); Assert.assertEquals(14L, extractor.extractTimestamp(14L, 0L)); Assert.assertEquals(20L, extractor.extractTimestamp(20L, 0L)); Assert.assertEquals(20L, extractor.extractTimestamp(20L, 0L)); Assert.assertEquals(20L, extractor.extractTimestamp(20L, 0L)); Assert.assertEquals(500L, extractor.extractTimestamp(500L, 0L)); Assert.assertEquals(9223372036854775806L, extractor.extractTimestamp(9223372036854775806L, 99999L)); } private void runInvalidTest(AscendingTimestampExtractor<Long> extractor) { Assert.assertEquals(1000L, extractor.extractTimestamp(1000L, 100L)); Assert.assertEquals(1000L, extractor.extractTimestamp(1000L, 100L)); Assert.assertEquals(999L, extractor.extractTimestamp(999L, 100L)); } private static class LongExtractor extends AscendingTimestampExtractor<Long> { private static final long serialVersionUID = 1L; private LongExtractor() { } public long extractAscendingTimestamp(Long element) { return element; } }这里使用withViolationHandler设置了violationHandler为FailingHandler，在遇到999这个时间的时候，由于比之前的1000小，因而会调用MonotonyViolationHandler.handleViolation方法小结flink为了方便开发提供了几个内置的Pre-defined Timestamp Extractors / Watermark Emitters，其中一个就是AscendingTimestampExtractorAscendingTimestampExtractor抽象类实现AssignerWithPeriodicWatermarks接口的extractTimestamp及getCurrentWatermark方法，同时声明抽象方法extractAscendingTimestamp供子类实现AscendingTimestampExtractor适用于elements的时间在每个parallel task里头是单调递增的，对于违反timestamp monotony的，这里调用MonotonyViolationHandler的handleViolation方法进行处理；MonotonyViolationHandler继承了Serializable，它定义了handleViolation方法，这个接口内置有三个实现类，分别是IgnoringHandler、FailingHandler、FailingHandlerdocPre-defined Timestamp Extractors / Watermark Emitters