[Debian-iot-maintainers] Bug#1138305: libcoap3: FTBFS with openssl 4.0
Ravi Kant Sharma
ravi.kant.sharma at canonical.com
Mon Aug 24 22:46:24 BST 2026
Package: libcoap3
Followup-For: Bug #1138305
X-Debbugs-Cc: ravi.kant.sharma at canonical.com, sebastian at breakpoint.cc
Control: tags -1 patch ftbfs
Dear Maintainer,
I figured why the build failed. The commit (https://github.com/obgm/libcoap/commit/5ea8b7d5e956d1574d78115da007bf23d118f1ae) is not sufficient. There is another commit which touches the same file https://github.com/obgm/libcoap/blob/84560ef8250d5a78417c98428f767c489b9b46b9/src/coap_openssl.c. I have attached my backported patch and have successfully tested the build: https://launchpad.net/~ravi-sharma/+archive/ubuntu/fix-lp2154852-v3/+packages.
-- System Information:
Debian Release: trixie/sid
APT prefers noble-updates
APT policy: (500, 'noble-updates'), (500, 'noble-security'), (500, 'noble'), (100, 'noble-backports')
Architecture: amd64 (x86_64)
Foreign Architectures: i386
Kernel: Linux 6.8.0-136-generic (SMP w/12 CPU threads; PREEMPT)
Kernel taint flags: TAINT_PROPRIETARY_MODULE, TAINT_WARN, TAINT_OOT_MODULE
Locale: LANG=en_GB.UTF-8, LC_CTYPE=en_GB.UTF-8 (charmap=UTF-8), LANGUAGE not set
Shell: /bin/sh linked to /usr/bin/dash
Init: systemd (via /run/systemd/system)
LSM: AppArmor: enabled
-------------- next part --------------
Subject: [PATCH] OpenSSL: Support 4.0 and later buids
Remove all references to ENGINE_*() as support has been removed in 4.0.
There currently is no tested Provider support for PKCS11.
Origin: backport, https://github.com/obgm/libcoap/commit/5ea8b7d5e956d1574d78115da007bf23d118f1ae
Bug-Ubuntu: https://launchpad.net/bugs/2154852
Bug-Debian: https://bugs.debian.org/1138305
Last-Update: 2026-08-21
---
src/coap_openssl.c | 208 +++++++++++++++++++++++++--------------------
1 file changed, 116 insertions(+), 92 deletions(-)
Index: libcoap3/src/coap_openssl.c
===================================================================
--- libcoap3.orig/src/coap_openssl.c 2026-08-24 13:10:48.095052175 +0200
+++ libcoap3/src/coap_openssl.c 2026-08-24 13:12:07.215294378 +0200
@@ -67,9 +67,12 @@
* The PKCSLL engine ID can be defined by (example)
+ -DCOAP_OPENSSL_PKCS11_ENGINE_ID='\"pkcs11\"'
*
+ * ENINE_* functions are no longer supported by OpenSSL 4.0 and later.
*/
#include <openssl/ssl.h>
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
#include <openssl/engine.h>
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
#include <openssl/err.h>
#include <openssl/rand.h>
#include <openssl/hmac.h>
@@ -244,7 +247,11 @@
*/
int
coap_dtls_pkcs11_is_supported(void) {
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
return 1;
+#else /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
+ return 0;
+#endif /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
}
/*
@@ -283,18 +290,23 @@
return &version;
}
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
static ENGINE *pkcs11_engine = NULL;
static ENGINE *defined_engine = NULL;
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
void
coap_dtls_startup(void) {
SSL_load_error_strings();
SSL_library_init();
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
ENGINE_load_dynamic();
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
}
void
coap_dtls_shutdown(void) {
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (pkcs11_engine) {
/* Release the functional reference from ENGINE_init() */
ENGINE_finish(pkcs11_engine);
@@ -305,6 +317,7 @@
ENGINE_finish(defined_engine);
defined_engine = NULL;
}
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
ERR_free_strings();
coap_dtls_set_log_level(COAP_LOG_EMERG);
}
@@ -320,6 +333,7 @@
return NULL;
}
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
static int
get_split_conf_entry(const uint8_t **start, size_t size, const char *get_keyword,
coap_str_const_t **p1, coap_str_const_t **p2) {
@@ -541,6 +555,19 @@
}
return 0;
}
+#else /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
+
+int
+coap_tls_engine_configure(coap_str_const_t *conf_mem) {
+ (void)conf_mem;
+ return 0;
+}
+
+int
+coap_tls_engine_remove(void) {
+ return 0;
+}
+#endif /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
/*
* Logging levels use the standard CoAP logging levels
@@ -1393,6 +1420,7 @@
}
}
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
static X509 *
missing_ENGINE_load_cert(ENGINE *engine, const char *cert_id) {
struct {
@@ -1441,6 +1469,7 @@
}
return 1;
}
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
#if OPENSSL_VERSION_NUMBER < 0x10101000L && COAP_SERVER_SUPPORT
@@ -1874,10 +1903,11 @@
return 1;
}
-#endif /* OPENSSL_VERSION_NUMBER < 0x10101000L */
+#endif /* OPENSSL_VERSION_NUMBER < 0x10101000L && COAP_SERVER_SUPPORT */
#if OPENSSL_VERSION_NUMBER >= 0x10101000L || COAP_CLIENT_SUPPORT
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
static int
install_engine_public_cert(ENGINE *engine, SSL *ssl, const char *public_cert,
coap_dtls_role_t role) {
@@ -1965,6 +1995,7 @@
X509_free(x509);
return 1;
}
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
static int
load_in_cas(SSL *ssl,
@@ -2027,9 +2058,7 @@
if (!(SSL_use_PrivateKey_file(ssl,
key.key.define.private_key.s_byte,
SSL_FILETYPE_PEM))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
break;
case COAP_PKI_KEY_DEF_PEM_BUF: /* define private key */
@@ -2043,31 +2072,23 @@
BIO_free(bp);
if (pkey)
EVP_PKEY_free(pkey);
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
if (bp)
BIO_free(bp);
if (pkey)
EVP_PKEY_free(pkey);
} else {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_NONE,
- &key, role, 0);
+ goto fail_none_private;
}
break;
case COAP_PKI_KEY_DEF_RPK_BUF: /* define private key */
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_private;
case COAP_PKI_KEY_DEF_DER: /* define private key */
if (!(SSL_use_PrivateKey_file(ssl,
key.key.define.private_key.s_byte,
SSL_FILETYPE_ASN1))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
break;
case COAP_PKI_KEY_DEF_DER_BUF: /* define private key */
@@ -2076,14 +2097,13 @@
ssl,
key.key.define.private_key.u_byte,
(long)key.key.define.private_key_len))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
break;
case COAP_PKI_KEY_DEF_PKCS11: /* define private key */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!check_pkcs11_engine()) {
- return 0;
+ goto fail_bad_private;
}
if (key.key.define.user_pin) {
/* If not set, pin-value may be held in pkcs11: URI */
@@ -2092,39 +2112,36 @@
key.key.define.user_pin, 0) == 0) {
coap_log_warn("*** setup_pki: (D)TLS: PKCS11: %s: Unable to set pin\n",
key.key.define.user_pin);
- return 0;
+ goto fail_bad_private;
}
}
if (!install_engine_private_key(pkcs11_engine, ssl,
key.key.define.private_key.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
+#else /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
+ goto fail_bad_private;
+#endif /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
break;
case COAP_PKI_KEY_DEF_ENGINE: /* define private key */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!defined_engine ||
!install_engine_private_key(defined_engine, ssl,
key.key.define.private_key.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_private;
}
break;
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
case COAP_PKI_KEY_DEF_PKCS11_RPK: /* define private key */
default:
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_private;
}
} else if (role == COAP_DTLS_ROLE_SERVER ||
(key.key.define.public_cert.u_byte &&
key.key.define.public_cert.u_byte[0])) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
- COAP_DEFINE_FAIL_NONE,
- &key, role, 0);
+ goto fail_none_private;
}
/*
@@ -2153,31 +2170,23 @@
BIO_free(bp);
if (cert)
X509_free(cert);
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
if (bp)
BIO_free(bp);
if (cert)
X509_free(cert);
} else {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
break;
case COAP_PKI_KEY_DEF_RPK_BUF: /* define public cert */
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_public;
case COAP_PKI_KEY_DEF_DER: /* define public cert */
if (!(SSL_use_certificate_file(ssl,
key.key.define.public_cert.s_byte,
SSL_FILETYPE_ASN1))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
break;
case COAP_PKI_KEY_DEF_DER_BUF: /* define public cert */
@@ -2185,42 +2194,41 @@
!(SSL_use_certificate_ASN1(ssl,
key.key.define.public_cert.u_byte,
(int)key.key.define.public_cert_len))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
break;
case COAP_PKI_KEY_DEF_PKCS11: /* define public cert */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!check_pkcs11_engine()) {
- return 0;
+ goto fail_bad_public;
}
if (!install_engine_public_cert(pkcs11_engine, ssl,
key.key.define.public_cert.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
+#else /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
+ goto fail_bad_public;
+#endif /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
break;
case COAP_PKI_KEY_DEF_ENGINE: /* define public cert */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!defined_engine ||
!install_engine_public_cert(defined_engine, ssl,
key.key.define.public_cert.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_public;
}
break;
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
case COAP_PKI_KEY_DEF_PKCS11_RPK: /* define public cert */
default:
- return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_public;
}
} else if (role == COAP_DTLS_ROLE_SERVER ||
(key.key.define.private_key.u_byte &&
key.key.define.private_key.u_byte[0])) {
+
return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
COAP_DEFINE_FAIL_NONE,
&key, role, 0);
@@ -2234,9 +2242,7 @@
switch (key.key.define.ca_def) {
case COAP_PKI_KEY_DEF_PEM:
if (!load_in_cas(ssl, key.key.define.ca.s_byte, role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
break;
case COAP_PKI_KEY_DEF_PEM_BUF: /* define ca */
@@ -2259,22 +2265,16 @@
BIO_free(bp);
}
} else {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
break;
case COAP_PKI_KEY_DEF_RPK_BUF: /* define ca */
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_ca;
case COAP_PKI_KEY_DEF_DER: /* define ca */
if (!(SSL_use_certificate_file(ssl,
key.key.define.ca.s_byte,
SSL_FILETYPE_ASN1))) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
break;
case COAP_PKI_KEY_DEF_DER_BUF: /* define ca */
@@ -2288,9 +2288,7 @@
if (role == COAP_DTLS_ROLE_SERVER) {
if (!x509 || !SSL_add_client_CA(ssl, x509)) {
X509_free(x509);
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
}
@@ -2302,36 +2300,66 @@
}
break;
case COAP_PKI_KEY_DEF_PKCS11: /* define ca */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!check_pkcs11_engine()) {
- return 0;
+ goto fail_bad_ca;
}
if (!install_engine_ca(pkcs11_engine, ssl,
key.key.define.ca.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
+#else /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
+ goto fail_bad_ca;
+#endif /* OPENSSL_VERSION_NUMBER >= 0x40000000L */
break;
case COAP_PKI_KEY_DEF_ENGINE: /* define ca */
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (!defined_engine ||
!install_engine_ca(defined_engine, ssl,
key.key.define.ca.s_byte,
role)) {
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_BAD,
- &key, role, 0);
+ goto fail_bad_ca;
}
break;
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
case COAP_PKI_KEY_DEF_PKCS11_RPK: /* define ca */
default:
- return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
- COAP_DEFINE_FAIL_NOT_SUPPORTED,
- &key, role, 0);
+ goto fail_ns_ca;
}
}
return 1;
+
+fail_bad_private:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
+ COAP_DEFINE_FAIL_BAD,
+ &key, role, 0);
+fail_ns_private:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
+ COAP_DEFINE_FAIL_NOT_SUPPORTED,
+ &key, role, 0);
+fail_none_private:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_PRIVATE,
+ COAP_DEFINE_FAIL_NONE,
+ &key, role, 0);
+fail_bad_public:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
+ COAP_DEFINE_FAIL_BAD,
+ &key, role, 0);
+fail_ns_public:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_PUBLIC,
+ COAP_DEFINE_FAIL_NOT_SUPPORTED,
+ &key, role, 0);
+fail_bad_ca:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
+ COAP_DEFINE_FAIL_BAD,
+ &key, role, 0);
+fail_ns_ca:
+ return coap_dtls_define_issue(COAP_DEFINE_KEY_CA,
+ COAP_DEFINE_FAIL_NOT_SUPPORTED,
+ &key, role, 0);
+
}
#endif /* OPENSSL_VERSION_NUMBER >= 0x10101000L || COAP_CLIENT_SUPPORT */
@@ -3158,6 +3186,7 @@
return 0;
context->setup_data = *setup_data;
+#if OPENSSL_VERSION_NUMBER < 0x40000000L
if (context->setup_data.pki_key.key_type == COAP_PKI_KEY_DEFINE) {
if (context->setup_data.pki_key.key.define.ca_def == COAP_PKI_KEY_DEF_ENGINE ||
context->setup_data.pki_key.key.define.public_cert_def == COAP_PKI_KEY_DEF_ENGINE ||
@@ -3168,6 +3197,7 @@
}
}
}
+#endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
if (!context->setup_data.verify_peer_cert) {
/* Needs to be clear so that no CA DNs are transmitted */
More information about the Debian-iot-maintainers
mailing list