diff --git a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts index a6f65cc23af39..f34c956e8922d 100644 --- a/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts +++ b/build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts @@ -171,6 +171,18 @@ tasks.withType().configureEach { "-XX:+EnableDynamicAgentLoading", "-Xshare:off", "-Dio.netty.tryReflectionSetAccessible=true", + // Netty 4.2 changed the default SslContextBuilder.endpointIdentificationAlgorithm + // from null to "HTTPS", which enables hostname verification on every TLS client + // connection that does not explicitly set the algorithm. Pulsar's TLS tests were + // written against the 4.1 default (no verification unless explicitly configured), + // and several of them either use certificates without matching hostnames or + // deliberately exercise "hostname verification disabled" paths. Restore the 4.1 + // default for the test JVM so those tests keep running. Production launch scripts + // are deliberately not changed; a follow-up PR should audit Pulsar's + // SslContextBuilder call sites and explicitly set endpointIdentificationAlgorithm + // based on the user's hostnameVerification configuration, after which this test + // override can be removed. + "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE", "-Dpulsar.allocator.pooled=true", "-Dpulsar.allocator.exit_on_oom=false", "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap", diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 351d4884548fa..a19f29095facf 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -293,26 +293,27 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.20.0.jar - org.apache.commons-commons-text-1.14.0.jar * Netty - - io.netty-netty-buffer-4.1.132.Final.jar - - io.netty-netty-codec-4.1.132.Final.jar - - io.netty-netty-codec-dns-4.1.132.Final.jar - - io.netty-netty-codec-http-4.1.132.Final.jar - - io.netty-netty-codec-http2-4.1.132.Final.jar - - io.netty-netty-codec-socks-4.1.132.Final.jar - - io.netty-netty-codec-haproxy-4.1.132.Final.jar - - io.netty-netty-common-4.1.132.Final.jar - - io.netty-netty-handler-4.1.132.Final.jar - - io.netty-netty-handler-proxy-4.1.132.Final.jar - - io.netty-netty-resolver-4.1.132.Final.jar - - io.netty-netty-resolver-dns-4.1.132.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.132.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.132.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.132.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.132.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.132.Final.jar - - io.netty-netty-transport-native-epoll-4.1.132.Final-linux-aarch_64.jar - - io.netty-netty-transport-native-epoll-4.1.132.Final-linux-x86_64.jar - - io.netty-netty-transport-native-unix-common-4.1.132.Final.jar + - io.netty-netty-buffer-4.2.12.Final.jar + - io.netty-netty-codec-base-4.2.12.Final.jar + - io.netty-netty-codec-compression-4.2.12.Final.jar + - io.netty-netty-codec-dns-4.2.12.Final.jar + - io.netty-netty-codec-http-4.2.12.Final.jar + - io.netty-netty-codec-http2-4.2.12.Final.jar + - io.netty-netty-codec-socks-4.2.12.Final.jar + - io.netty-netty-codec-haproxy-4.2.12.Final.jar + - io.netty-netty-common-4.2.12.Final.jar + - io.netty-netty-handler-4.2.12.Final.jar + - io.netty-netty-handler-proxy-4.2.12.Final.jar + - io.netty-netty-resolver-4.2.12.Final.jar + - io.netty-netty-resolver-dns-4.2.12.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.2.12.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.2.12.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.2.12.Final-osx-x86_64.jar + - io.netty-netty-transport-4.2.12.Final.jar + - io.netty-netty-transport-classes-epoll-4.2.12.Final.jar + - io.netty-netty-transport-native-epoll-4.2.12.Final-linux-aarch_64.jar + - io.netty-netty-transport-native-epoll-4.2.12.Final-linux-x86_64.jar + - io.netty-netty-transport-native-unix-common-4.2.12.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.75.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-linux-x86_64.jar @@ -320,6 +321,9 @@ The Apache Software License, Version 2.0 - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-osx-x86_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-windows-x86_64.jar - io.netty-netty-tcnative-classes-2.0.75.Final.jar + - io.netty-netty-transport-classes-io_uring-4.2.12.Final.jar + - io.netty-netty-transport-native-io_uring-4.2.12.Final-linux-x86_64.jar + - io.netty-netty-transport-native-io_uring-4.2.12.Final-linux-aarch_64.jar - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 350f38dcdadc6..7aec9549e20dc 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -345,22 +345,23 @@ The Apache Software License, Version 2.0 - commons-text-1.14.0.jar - commons-compress-1.28.0.jar * Netty - - netty-buffer-4.1.132.Final.jar - - netty-codec-4.1.132.Final.jar - - netty-codec-dns-4.1.132.Final.jar - - netty-codec-http-4.1.132.Final.jar - - netty-codec-socks-4.1.132.Final.jar - - netty-codec-haproxy-4.1.132.Final.jar - - netty-common-4.1.132.Final.jar - - netty-handler-4.1.132.Final.jar - - netty-handler-proxy-4.1.132.Final.jar - - netty-resolver-4.1.132.Final.jar - - netty-resolver-dns-4.1.132.Final.jar - - netty-transport-4.1.132.Final.jar - - netty-transport-classes-epoll-4.1.132.Final.jar - - netty-transport-native-epoll-4.1.132.Final-linux-aarch_64.jar - - netty-transport-native-epoll-4.1.132.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.132.Final.jar + - netty-buffer-4.2.12.Final.jar + - netty-codec-base-4.2.12.Final.jar + - netty-codec-compression-4.2.12.Final.jar + - netty-codec-dns-4.2.12.Final.jar + - netty-codec-http-4.2.12.Final.jar + - netty-codec-socks-4.2.12.Final.jar + - netty-codec-haproxy-4.2.12.Final.jar + - netty-common-4.2.12.Final.jar + - netty-handler-4.2.12.Final.jar + - netty-handler-proxy-4.2.12.Final.jar + - netty-resolver-4.2.12.Final.jar + - netty-resolver-dns-4.2.12.Final.jar + - netty-transport-4.2.12.Final.jar + - netty-transport-classes-epoll-4.2.12.Final.jar + - netty-transport-native-epoll-4.2.12.Final-linux-aarch_64.jar + - netty-transport-native-epoll-4.2.12.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.2.12.Final.jar - netty-tcnative-boringssl-static-2.0.75.Final.jar - netty-tcnative-boringssl-static-2.0.75.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.75.Final-linux-x86_64.jar @@ -368,12 +369,12 @@ The Apache Software License, Version 2.0 - netty-tcnative-boringssl-static-2.0.75.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.75.Final-windows-x86_64.jar - netty-tcnative-classes-2.0.75.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar - - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar - - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.132.Final.jar - - netty-resolver-dns-native-macos-4.1.132.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.132.Final-osx-x86_64.jar + - netty-transport-classes-io_uring-4.2.12.Final.jar + - netty-transport-native-io_uring-4.2.12.Final-linux-aarch_64.jar + - netty-transport-native-io_uring-4.2.12.Final-linux-x86_64.jar + - netty-resolver-dns-classes-macos-4.2.12.Final.jar + - netty-resolver-dns-native-macos-4.2.12.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.2.12.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b5a5aca4a675d..b6a087ec32ae9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -25,8 +25,7 @@ checkstyle = "13.3.0" # Major frameworks bookkeeper = "4.17.3" zookeeper = "3.9.5" -netty = "4.1.132.Final" -netty-iouring = "0.0.26.Final" +netty = "4.2.12.Final" jetty = "12.1.6" jersey = "2.42" jackson = "2.21.2" @@ -215,8 +214,8 @@ netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" } netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common", version.ref = "netty" } netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" } -netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" } -netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" } +netty-transport-classes-io_uring = { module = "io.netty:netty-transport-classes-io_uring", version.ref = "netty" } +netty-transport-native-io-uring = { module = "io.netty:netty-transport-native-io_uring", version.ref = "netty" } netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" } # Protobuf / gRPC protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref = "protobuf" } diff --git a/pulsar-common/build.gradle.kts b/pulsar-common/build.gradle.kts index 5388569ee135e..71a2fd16a1f5e 100644 --- a/pulsar-common/build.gradle.kts +++ b/pulsar-common/build.gradle.kts @@ -122,9 +122,9 @@ dependencies { implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("linux-aarch_64") }) implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("osx-x86_64") }) implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("osx-aarch_64") }) - implementation(libs.netty.incubator.transport.classes.io.uring) - implementation(variantOf(libs.netty.incubator.transport.native.io.uring) { classifier("linux-x86_64") }) - implementation(variantOf(libs.netty.incubator.transport.native.io.uring) { classifier("linux-aarch_64") }) + implementation(libs.netty.transport.classes.io.uring) + implementation(variantOf(libs.netty.transport.native.io.uring) { classifier("linux-x86_64") }) + implementation(variantOf(libs.netty.transport.native.io.uring) { classifier("linux-aarch_64") }) implementation(libs.netty.codec.haproxy) implementation(libs.commons.lang3) implementation(libs.jakarta.ws.rs.api) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/EventLoopUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/EventLoopUtil.java index 3dbdd28871532..f0e3d9cba641c 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/EventLoopUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/EventLoopUtil.java @@ -20,6 +20,7 @@ import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; import io.netty.channel.SelectStrategy; import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollChannelOption; @@ -35,11 +36,11 @@ import io.netty.channel.socket.nio.NioDatagramChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; -import io.netty.incubator.channel.uring.IOUring; -import io.netty.incubator.channel.uring.IOUringDatagramChannel; -import io.netty.incubator.channel.uring.IOUringEventLoopGroup; -import io.netty.incubator.channel.uring.IOUringServerSocketChannel; -import io.netty.incubator.channel.uring.IOUringSocketChannel; +import io.netty.channel.uring.IoUring; +import io.netty.channel.uring.IoUringDatagramChannel; +import io.netty.channel.uring.IoUringIoHandler; +import io.netty.channel.uring.IoUringServerSocketChannel; +import io.netty.channel.uring.IoUringSocketChannel; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadFactory; import lombok.CustomLog; @@ -52,13 +53,25 @@ public class EventLoopUtil { private static final String ENABLE_IO_URING = "pulsar.enableUring"; + /** + * Marker subclass so we can distinguish io_uring-backed event loops via instanceof. + * Netty 4.2 removed the dedicated {@code IOUringEventLoopGroup} class in favor of the + * generic {@link MultiThreadIoEventLoopGroup} + {@link IoUringIoHandler} pair, which + * makes io_uring groups indistinguishable by type from any other MultiThreadIoEventLoopGroup. + */ + private static final class IoUringMultiThreadIoEventLoopGroup extends MultiThreadIoEventLoopGroup { + IoUringMultiThreadIoEventLoopGroup(int nThreads, ThreadFactory threadFactory) { + super(nThreads, threadFactory, IoUringIoHandler.newFactory()); + } + } + /** * @return an EventLoopGroup suitable for the current platform */ public static EventLoopGroup newEventLoopGroup(int nThreads, boolean enableBusyWait, ThreadFactory threadFactory) { if (Epoll.isAvailable()) { if (isIoUringEnabledAndAvailable()) { - return new IOUringEventLoopGroup(nThreads, threadFactory); + return new IoUringMultiThreadIoEventLoopGroup(nThreads, threadFactory); } else { if (!enableBusyWait) { // Regular Epoll based event loop @@ -96,8 +109,8 @@ private static boolean isIoUringEnabledAndAvailable() { String ioUringSetting = System.getProperty(ENABLE_IO_URING); boolean ioUringEnabled = "1".equalsIgnoreCase(ioUringSetting) || "true".equalsIgnoreCase(ioUringSetting); if (ioUringEnabled) { - // Throw exception if IOUring cannot be used - IOUring.ensureAvailability(); + // Throw exception if IoUring cannot be used + IoUring.ensureAvailability(); } return ioUringEnabled; } @@ -109,8 +122,8 @@ private static boolean isIoUringEnabledAndAvailable() { * @return */ public static Class getClientSocketChannelClass(EventLoopGroup eventLoopGroup) { - if (eventLoopGroup instanceof IOUringEventLoopGroup) { - return IOUringSocketChannel.class; + if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) { + return IoUringSocketChannel.class; } else if (eventLoopGroup instanceof EpollEventLoopGroup) { return EpollSocketChannel.class; } else { @@ -128,7 +141,7 @@ public static Class getClientSocketChannelClass(EventLo public static Class getClientSocketChannelClass() { if (Epoll.isAvailable()) { if (isIoUringEnabledAndAvailable()) { - return IOUringSocketChannel.class; + return IoUringSocketChannel.class; } else { return EpollSocketChannel.class; } @@ -138,8 +151,8 @@ public static Class getClientSocketChannelClass() { } public static Class getServerSocketChannelClass(EventLoopGroup eventLoopGroup) { - if (eventLoopGroup instanceof IOUringEventLoopGroup) { - return IOUringServerSocketChannel.class; + if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) { + return IoUringServerSocketChannel.class; } else if (eventLoopGroup instanceof EpollEventLoopGroup) { return EpollServerSocketChannel.class; } else { @@ -157,7 +170,7 @@ public static Class getServerSocketChannelClass(E public static Class getServerSocketChannelClass() { if (Epoll.isAvailable()) { if (isIoUringEnabledAndAvailable()) { - return IOUringServerSocketChannel.class; + return IoUringServerSocketChannel.class; } else { return EpollServerSocketChannel.class; } @@ -167,8 +180,8 @@ public static Class getServerSocketChannelClass() } public static Class getDatagramChannelClass(EventLoopGroup eventLoopGroup) { - if (eventLoopGroup instanceof IOUringEventLoopGroup) { - return IOUringDatagramChannel.class; + if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) { + return IoUringDatagramChannel.class; } else if (eventLoopGroup instanceof EpollEventLoopGroup) { return EpollDatagramChannel.class; } else { @@ -186,7 +199,7 @@ public static Class getDatagramChannelClass(EventLoop public static Class getDatagramChannelClass() { if (Epoll.isAvailable()) { if (isIoUringEnabledAndAvailable()) { - return IOUringDatagramChannel.class; + return IoUringDatagramChannel.class; } else { return EpollDatagramChannel.class; } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java index 8061f853d66c1..b0bbd1ce0308e 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java @@ -28,11 +28,13 @@ public void testRecycle() { BitSetRecyclable bitset1 = BitSetRecyclable.create(); bitset1.set(3); bitset1.recycle(); + // Netty 4.2's Recycler (which is itself deprecated) no longer guarantees same-thread + // immediate reuse, so we only assert on functional behavior: any recycled instance + // must come back cleared, and distinct create() calls must return distinct objects. BitSetRecyclable bitset2 = BitSetRecyclable.create(); BitSetRecyclable bitset3 = BitSetRecyclable.create(); - Assert.assertSame(bitset2, bitset1); Assert.assertFalse(bitset2.get(3)); - Assert.assertNotSame(bitset3, bitset1); + Assert.assertNotSame(bitset3, bitset2); } @Test diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentBitSetRecyclableTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentBitSetRecyclableTest.java index 7ffda74156969..605e0115a20c6 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentBitSetRecyclableTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentBitSetRecyclableTest.java @@ -30,11 +30,13 @@ public void testRecycle() { ConcurrentBitSetRecyclable bitset1 = ConcurrentBitSetRecyclable.create(); bitset1.set(3); bitset1.recycle(); + // Netty 4.2's Recycler (which is itself deprecated) no longer guarantees same-thread + // immediate reuse, so we only assert on functional behavior: any recycled instance + // must come back cleared, and distinct create() calls must return distinct objects. ConcurrentBitSetRecyclable bitset2 = ConcurrentBitSetRecyclable.create(); ConcurrentBitSetRecyclable bitset3 = ConcurrentBitSetRecyclable.create(); - Assert.assertSame(bitset2, bitset1); Assert.assertFalse(bitset2.get(3)); - Assert.assertNotSame(bitset3, bitset1); + Assert.assertNotSame(bitset3, bitset2); } @SuppressWarnings("deprecation")