[improve][build] Upgrade Netty from 4.1.132.Final to 4.2.12.Final

build-logic/conventions/src/main/kotlin/pulsar.java-conventions.gradle.kts

@@ -171,6 +171,18 @@ tasks.withType<Test>().configureEach {
         "-XX:+EnableDynamicAgentLoading",
         "-Xshare:off",
         "-Dio.netty.tryReflectionSetAccessible=true",
+        // Netty 4.2 changed the default SslContextBuilder.endpointIdentificationAlgorithm
+        // from null to "HTTPS", which enables hostname verification on every TLS client
+        // connection that does not explicitly set the algorithm. Pulsar's TLS tests were
+        // written against the 4.1 default (no verification unless explicitly configured),
+        // and several of them either use certificates without matching hostnames or
+        // deliberately exercise "hostname verification disabled" paths. Restore the 4.1
+        // default for the test JVM so those tests keep running. Production launch scripts
+        // are deliberately not changed; a follow-up PR should audit Pulsar's
+        // SslContextBuilder call sites and explicitly set endpointIdentificationAlgorithm
+        // based on the user's hostnameVerification configuration, after which this test
+        // override can be removed.
+        "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE",
         "-Dpulsar.allocator.pooled=true",
         "-Dpulsar.allocator.exit_on_oom=false",
         "-Dpulsar.allocator.out_of_memory_policy=FallbackToHeap",

distribution/server/src/assemble/LICENSE.bin.txt

@@ -293,33 +293,37 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.20.0.jar
     - org.apache.commons-commons-text-1.14.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.132.Final.jar
-    - io.netty-netty-codec-4.1.132.Final.jar
-    - io.netty-netty-codec-dns-4.1.132.Final.jar
-    - io.netty-netty-codec-http-4.1.132.Final.jar
-    - io.netty-netty-codec-http2-4.1.132.Final.jar
-    - io.netty-netty-codec-socks-4.1.132.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.132.Final.jar
-    - io.netty-netty-common-4.1.132.Final.jar
-    - io.netty-netty-handler-4.1.132.Final.jar
-    - io.netty-netty-handler-proxy-4.1.132.Final.jar
-    - io.netty-netty-resolver-4.1.132.Final.jar
-    - io.netty-netty-resolver-dns-4.1.132.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.132.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.132.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.132.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.132.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.132.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.132.Final-linux-aarch_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.132.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-unix-common-4.1.132.Final.jar
+    - io.netty-netty-buffer-4.2.12.Final.jar
+    - io.netty-netty-codec-base-4.2.12.Final.jar
+    - io.netty-netty-codec-compression-4.2.12.Final.jar
+    - io.netty-netty-codec-dns-4.2.12.Final.jar
+    - io.netty-netty-codec-http-4.2.12.Final.jar
+    - io.netty-netty-codec-http2-4.2.12.Final.jar
+    - io.netty-netty-codec-socks-4.2.12.Final.jar
+    - io.netty-netty-codec-haproxy-4.2.12.Final.jar
+    - io.netty-netty-common-4.2.12.Final.jar
+    - io.netty-netty-handler-4.2.12.Final.jar
+    - io.netty-netty-handler-proxy-4.2.12.Final.jar
+    - io.netty-netty-resolver-4.2.12.Final.jar
+    - io.netty-netty-resolver-dns-4.2.12.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.2.12.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.2.12.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.2.12.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.2.12.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.2.12.Final.jar
+    - io.netty-netty-transport-native-epoll-4.2.12.Final-linux-aarch_64.jar
+    - io.netty-netty-transport-native-epoll-4.2.12.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-unix-common-4.2.12.Final.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-linux-aarch_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-linux-x86_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-osx-aarch_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-osx-x86_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.75.Final-windows-x86_64.jar
     - io.netty-netty-tcnative-classes-2.0.75.Final.jar
+    - io.netty-netty-transport-classes-io_uring-4.2.12.Final.jar
+    - io.netty-netty-transport-native-io_uring-4.2.12.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-io_uring-4.2.12.Final-linux-aarch_64.jar
     - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar

distribution/shell/src/assemble/LICENSE.bin.txt

@@ -345,35 +345,36 @@ The Apache Software License, Version 2.0
     - commons-text-1.14.0.jar
     - commons-compress-1.28.0.jar
  * Netty
-    - netty-buffer-4.1.132.Final.jar
-    - netty-codec-4.1.132.Final.jar
-    - netty-codec-dns-4.1.132.Final.jar
-    - netty-codec-http-4.1.132.Final.jar
-    - netty-codec-socks-4.1.132.Final.jar
-    - netty-codec-haproxy-4.1.132.Final.jar
-    - netty-common-4.1.132.Final.jar
-    - netty-handler-4.1.132.Final.jar
-    - netty-handler-proxy-4.1.132.Final.jar
-    - netty-resolver-4.1.132.Final.jar
-    - netty-resolver-dns-4.1.132.Final.jar
-    - netty-transport-4.1.132.Final.jar
-    - netty-transport-classes-epoll-4.1.132.Final.jar
-    - netty-transport-native-epoll-4.1.132.Final-linux-aarch_64.jar
-    - netty-transport-native-epoll-4.1.132.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.132.Final.jar
+    - netty-buffer-4.2.12.Final.jar
+    - netty-codec-base-4.2.12.Final.jar
+    - netty-codec-compression-4.2.12.Final.jar
+    - netty-codec-dns-4.2.12.Final.jar
+    - netty-codec-http-4.2.12.Final.jar
+    - netty-codec-socks-4.2.12.Final.jar
+    - netty-codec-haproxy-4.2.12.Final.jar
+    - netty-common-4.2.12.Final.jar
+    - netty-handler-4.2.12.Final.jar
+    - netty-handler-proxy-4.2.12.Final.jar
+    - netty-resolver-4.2.12.Final.jar
+    - netty-resolver-dns-4.2.12.Final.jar
+    - netty-transport-4.2.12.Final.jar
+    - netty-transport-classes-epoll-4.2.12.Final.jar
+    - netty-transport-native-epoll-4.2.12.Final-linux-aarch_64.jar
+    - netty-transport-native-epoll-4.2.12.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.2.12.Final.jar
     - netty-tcnative-boringssl-static-2.0.75.Final.jar
     - netty-tcnative-boringssl-static-2.0.75.Final-linux-aarch_64.jar
     - netty-tcnative-boringssl-static-2.0.75.Final-linux-x86_64.jar
     - netty-tcnative-boringssl-static-2.0.75.Final-osx-aarch_64.jar
     - netty-tcnative-boringssl-static-2.0.75.Final-osx-x86_64.jar
     - netty-tcnative-boringssl-static-2.0.75.Final-windows-x86_64.jar
     - netty-tcnative-classes-2.0.75.Final.jar
-    - netty-incubator-transport-classes-io_uring-0.0.26.Final.jar
-    - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-aarch_64.jar
-    - netty-incubator-transport-native-io_uring-0.0.26.Final-linux-x86_64.jar
-    - netty-resolver-dns-classes-macos-4.1.132.Final.jar
-    - netty-resolver-dns-native-macos-4.1.132.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.132.Final-osx-x86_64.jar
+    - netty-transport-classes-io_uring-4.2.12.Final.jar
+    - netty-transport-native-io_uring-4.2.12.Final-linux-aarch_64.jar
+    - netty-transport-native-io_uring-4.2.12.Final-linux-x86_64.jar
+    - netty-resolver-dns-classes-macos-4.2.12.Final.jar
+    - netty-resolver-dns-native-macos-4.2.12.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.2.12.Final-osx-x86_64.jar
  * Prometheus client
     - simpleclient-0.16.0.jar
     - simpleclient_log4j2-0.16.0.jar

gradle/libs.versions.toml

@@ -25,8 +25,7 @@ checkstyle = "13.3.0"
 # Major frameworks
 bookkeeper = "4.17.3"
 zookeeper = "3.9.5"
-netty = "4.1.132.Final"
-netty-iouring = "0.0.26.Final"
+netty = "4.2.12.Final"
 jetty = "12.1.6"
 jersey = "2.42"
 jackson = "2.21.2"
@@ -215,8 +214,8 @@ netty-resolver-dns-native-macos = { module = "io.netty:netty-resolver-dns-native
 netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" }
 netty-transport-native-unix-common = { module = "io.netty:netty-transport-native-unix-common", version.ref = "netty" }
 netty-tcnative-boringssl-static = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" }
-netty-incubator-transport-classes-io_uring = { module = "io.netty.incubator:netty-incubator-transport-classes-io_uring", version.ref = "netty-iouring" }
-netty-incubator-transport-native-io-uring = { module = "io.netty.incubator:netty-incubator-transport-native-io_uring", version.ref = "netty-iouring" }
+netty-transport-classes-io_uring = { module = "io.netty:netty-transport-classes-io_uring", version.ref = "netty" }
+netty-transport-native-io-uring = { module = "io.netty:netty-transport-native-io_uring", version.ref = "netty" }
 netty-reactive-streams = { module = "com.typesafe.netty:netty-reactive-streams", version.ref = "netty-reactive-streams" }
 # Protobuf / gRPC
 protobuf-bom = { module = "com.google.protobuf:protobuf-bom", version.ref = "protobuf" }

pulsar-common/build.gradle.kts

@@ -122,9 +122,9 @@ dependencies {
     implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("linux-aarch_64") })
     implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("osx-x86_64") })
     implementation(variantOf(libs.netty.tcnative.boringssl.static) { classifier("osx-aarch_64") })
-    implementation(libs.netty.incubator.transport.classes.io.uring)
-    implementation(variantOf(libs.netty.incubator.transport.native.io.uring) { classifier("linux-x86_64") })
-    implementation(variantOf(libs.netty.incubator.transport.native.io.uring) { classifier("linux-aarch_64") })
+    implementation(libs.netty.transport.classes.io.uring)
+    implementation(variantOf(libs.netty.transport.native.io.uring) { classifier("linux-x86_64") })
+    implementation(variantOf(libs.netty.transport.native.io.uring) { classifier("linux-aarch_64") })
     implementation(libs.netty.codec.haproxy)
     implementation(libs.commons.lang3)
     implementation(libs.jakarta.ws.rs.api)

pulsar-common/src/main/java/org/apache/pulsar/common/util/netty/EventLoopUtil.java

@@ -20,6 +20,7 @@
 
 import io.netty.bootstrap.ServerBootstrap;
 import io.netty.channel.EventLoopGroup;
+import io.netty.channel.MultiThreadIoEventLoopGroup;
 import io.netty.channel.SelectStrategy;
 import io.netty.channel.epoll.Epoll;
 import io.netty.channel.epoll.EpollChannelOption;
@@ -35,11 +36,11 @@
 import io.netty.channel.socket.nio.NioDatagramChannel;
 import io.netty.channel.socket.nio.NioServerSocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.incubator.channel.uring.IOUring;
-import io.netty.incubator.channel.uring.IOUringDatagramChannel;
-import io.netty.incubator.channel.uring.IOUringEventLoopGroup;
-import io.netty.incubator.channel.uring.IOUringServerSocketChannel;
-import io.netty.incubator.channel.uring.IOUringSocketChannel;
+import io.netty.channel.uring.IoUring;
+import io.netty.channel.uring.IoUringDatagramChannel;
+import io.netty.channel.uring.IoUringIoHandler;
+import io.netty.channel.uring.IoUringServerSocketChannel;
+import io.netty.channel.uring.IoUringSocketChannel;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadFactory;
 import lombok.CustomLog;
@@ -52,13 +53,25 @@ public class EventLoopUtil {
 
     private static final String ENABLE_IO_URING = "pulsar.enableUring";
 
+    /**
+     * Marker subclass so we can distinguish io_uring-backed event loops via instanceof.
+     * Netty 4.2 removed the dedicated {@code IOUringEventLoopGroup} class in favor of the
+     * generic {@link MultiThreadIoEventLoopGroup} + {@link IoUringIoHandler} pair, which
+     * makes io_uring groups indistinguishable by type from any other MultiThreadIoEventLoopGroup.
+     */
+    private static final class IoUringMultiThreadIoEventLoopGroup extends MultiThreadIoEventLoopGroup {
+        IoUringMultiThreadIoEventLoopGroup(int nThreads, ThreadFactory threadFactory) {
+            super(nThreads, threadFactory, IoUringIoHandler.newFactory());
+        }
+    }
+
     /**
      * @return an EventLoopGroup suitable for the current platform
      */
     public static EventLoopGroup newEventLoopGroup(int nThreads, boolean enableBusyWait, ThreadFactory threadFactory) {
         if (Epoll.isAvailable()) {
             if (isIoUringEnabledAndAvailable()) {
-                return new IOUringEventLoopGroup(nThreads, threadFactory);
+                return new IoUringMultiThreadIoEventLoopGroup(nThreads, threadFactory);
             } else {
                 if (!enableBusyWait) {
                     // Regular Epoll based event loop
@@ -96,8 +109,8 @@ private static boolean isIoUringEnabledAndAvailable() {
         String ioUringSetting = System.getProperty(ENABLE_IO_URING);
         boolean ioUringEnabled = "1".equalsIgnoreCase(ioUringSetting) || "true".equalsIgnoreCase(ioUringSetting);
         if (ioUringEnabled) {
-            // Throw exception if IOUring cannot be used
-            IOUring.ensureAvailability();
+            // Throw exception if IoUring cannot be used
+            IoUring.ensureAvailability();
         }
         return ioUringEnabled;
     }
@@ -109,8 +122,8 @@ private static boolean isIoUringEnabledAndAvailable() {
      * @return
      */
     public static Class<? extends SocketChannel> getClientSocketChannelClass(EventLoopGroup eventLoopGroup) {
-        if (eventLoopGroup instanceof IOUringEventLoopGroup) {
-            return IOUringSocketChannel.class;
+        if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) {
+            return IoUringSocketChannel.class;
         } else if (eventLoopGroup instanceof EpollEventLoopGroup) {
             return EpollSocketChannel.class;
         } else {
@@ -128,7 +141,7 @@ public static Class<? extends SocketChannel> getClientSocketChannelClass(EventLo
     public static Class<? extends SocketChannel> getClientSocketChannelClass() {
         if (Epoll.isAvailable()) {
             if (isIoUringEnabledAndAvailable()) {
-                return IOUringSocketChannel.class;
+                return IoUringSocketChannel.class;
             } else {
                 return EpollSocketChannel.class;
             }
@@ -138,8 +151,8 @@ public static Class<? extends SocketChannel> getClientSocketChannelClass() {
     }
 
     public static Class<? extends ServerSocketChannel> getServerSocketChannelClass(EventLoopGroup eventLoopGroup) {
-        if (eventLoopGroup instanceof IOUringEventLoopGroup) {
-            return IOUringServerSocketChannel.class;
+        if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) {
+            return IoUringServerSocketChannel.class;
         } else if (eventLoopGroup instanceof EpollEventLoopGroup) {
             return EpollServerSocketChannel.class;
         } else {
@@ -157,7 +170,7 @@ public static Class<? extends ServerSocketChannel> getServerSocketChannelClass(E
     public static Class<? extends ServerSocketChannel> getServerSocketChannelClass() {
         if (Epoll.isAvailable()) {
             if (isIoUringEnabledAndAvailable()) {
-                return IOUringServerSocketChannel.class;
+                return IoUringServerSocketChannel.class;
             } else {
                 return EpollServerSocketChannel.class;
             }
@@ -167,8 +180,8 @@ public static Class<? extends ServerSocketChannel> getServerSocketChannelClass()
     }
 
     public static Class<? extends DatagramChannel> getDatagramChannelClass(EventLoopGroup eventLoopGroup) {
-        if (eventLoopGroup instanceof IOUringEventLoopGroup) {
-            return IOUringDatagramChannel.class;
+        if (eventLoopGroup instanceof IoUringMultiThreadIoEventLoopGroup) {
+            return IoUringDatagramChannel.class;
         } else if (eventLoopGroup instanceof EpollEventLoopGroup) {
             return EpollDatagramChannel.class;
         } else {
@@ -186,7 +199,7 @@ public static Class<? extends DatagramChannel> getDatagramChannelClass(EventLoop
     public static Class<? extends DatagramChannel> getDatagramChannelClass() {
         if (Epoll.isAvailable()) {
             if (isIoUringEnabledAndAvailable()) {
-                return IOUringDatagramChannel.class;
+                return IoUringDatagramChannel.class;
             } else {
                 return EpollDatagramChannel.class;
             }

pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/BitSetRecyclableRecyclableTest.java

@@ -28,11 +28,13 @@ public void testRecycle() {
         BitSetRecyclable bitset1 = BitSetRecyclable.create();
         bitset1.set(3);
         bitset1.recycle();
+        // Netty 4.2's Recycler (which is itself deprecated) no longer guarantees same-thread
+        // immediate reuse, so we only assert on functional behavior: any recycled instance
+        // must come back cleared, and distinct create() calls must return distinct objects.
         BitSetRecyclable bitset2 = BitSetRecyclable.create();
         BitSetRecyclable bitset3 = BitSetRecyclable.create();
-        Assert.assertSame(bitset2, bitset1);
         Assert.assertFalse(bitset2.get(3));
-        Assert.assertNotSame(bitset3, bitset1);
+        Assert.assertNotSame(bitset3, bitset2);
     }
 
     @Test

pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentBitSetRecyclableTest.java

@@ -30,11 +30,13 @@ public void testRecycle() {
         ConcurrentBitSetRecyclable bitset1 = ConcurrentBitSetRecyclable.create();
         bitset1.set(3);
         bitset1.recycle();
+        // Netty 4.2's Recycler (which is itself deprecated) no longer guarantees same-thread
+        // immediate reuse, so we only assert on functional behavior: any recycled instance
+        // must come back cleared, and distinct create() calls must return distinct objects.
         ConcurrentBitSetRecyclable bitset2 = ConcurrentBitSetRecyclable.create();
         ConcurrentBitSetRecyclable bitset3 = ConcurrentBitSetRecyclable.create();
-        Assert.assertSame(bitset2, bitset1);
         Assert.assertFalse(bitset2.get(3));
-        Assert.assertNotSame(bitset3, bitset1);
+        Assert.assertNotSame(bitset3, bitset2);
     }
 
     @SuppressWarnings("deprecation")