[pkg-uWSGI-devel] Bug#989836: current Bullseye version lacks Transfer-Encoding: chunked support
Thomas Goirand
zigo at debian.org
Mon Jun 14 14:44:13 BST 2021
Package: uwsgi
Version: 2.0.19.1-7
Severity: important
Tags: patch
Hi Jonas,
As per the subject line, current version of UWSGI as in Bullseye/Sid,
doesn't have support for the Transfer-Encoding: chunked mode of HTTP
transfer, which is needed by OpenStack Glance and Swift. As a consequence,
all the queries that are using this HTTP transfer mode (which is not
setting a Content-Lenght) fails.
Attached is the "git format-patch" output when adding the 4 years old
upstream patch. We've tested it in production, and this fixes the troubles
we had when setting-up Glance or Swift over UWSGI (intead of Eventlet).
I'd really love to have this patch added to Bullseye. Would you let me
try to deal with the release team to get this in?
Cheers,
Thomas Goirand (zigo)
-------------- next part --------------
>From 722db2ea22eb454ed678bd6ff8b1c2f287df4802 Mon Sep 17 00:00:00 2001
From: Thomas Goirand <zigo at debian.org>
Date: Fri, 11 Jun 2021 11:09:19 +0200
Subject: [PATCH] * Add upstream patch to support Transfer-Encoding:
chuncked, necessary for OpenStack Glance and Swift over uwsgi.
---
debian/changelog | 9 +
.../Add_support_for_chunked_encoding.patch | 241 ++++++++++++++++++
debian/patches/series | 1 +
3 files changed, 251 insertions(+)
create mode 100644 debian/patches/Add_support_for_chunked_encoding.patch
diff --git a/debian/changelog b/debian/changelog
index 70389417..b0372124 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+uwsgi (2.0.19.1-7.1) UNRELEASED; urgency=medium
+
+ [ Thomas Goirand ]
+ * Non-maintainer upload.
+ * Add upstream patch to support Transfer-Encoding: chuncked, necessary for
+ OpenStack Glance and Swift over uwsgi.
+
+ -- Thomas Goirand <zigo at debian.org> Fri, 11 Jun 2021 11:08:33 +0200
+
uwsgi (2.0.19.1-7) unstable; urgency=medium
* add patch cherry-picked upstream
diff --git a/debian/patches/Add_support_for_chunked_encoding.patch b/debian/patches/Add_support_for_chunked_encoding.patch
new file mode 100644
index 00000000..496a1e75
--- /dev/null
+++ b/debian/patches/Add_support_for_chunked_encoding.patch
@@ -0,0 +1,241 @@
+Subject: preliminary implementation of #1428
+ This implements support for transfer-encoding: chuncked
+Author: Unbit <info at unbit.it>
+Date: Thu, 9 Nov 2017 16:40:44 +0100
+Last-Update: 2021-06-11
+
+Index: uwsgi/core/chunked.c
+===================================================================
+--- uwsgi.orig/core/chunked.c
++++ uwsgi/core/chunked.c
+@@ -83,6 +83,83 @@ static ssize_t uwsgi_chunked_readline(st
+
+ */
+
++struct uwsgi_buffer *uwsgi_chunked_read_smart(struct wsgi_request *wsgi_req, size_t len, int timeout) {
++ // check for buffer
++ if (!wsgi_req->body_chunked_buf)
++ wsgi_req->body_chunked_buf = uwsgi_buffer_new(uwsgi.page_size);
++ // first case: asking for all
++ if (!len) {
++ for(;;) {
++ size_t chunked_len = 0;
++ char *buf = uwsgi_chunked_read(wsgi_req, &chunked_len, timeout, 0);
++ if (chunked_len == 0) {
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(wsgi_req->body_chunked_buf->pos);
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ return ret;
++ }
++ if (uwsgi_buffer_append(wsgi_req->body_chunked_buf, buf, chunked_len)) {
++ return NULL;
++ }
++ }
++ }
++
++ // asking for littler part
++ if (len <= wsgi_req->body_chunked_buf->pos) {
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(len);
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, len)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, len);
++ return ret;
++ }
++
++ // more data required
++ size_t remains = len;
++ struct uwsgi_buffer *ret = uwsgi_buffer_new(remains);
++ if (wsgi_req->body_chunked_buf->pos > 0) {
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ remains -= wsgi_req->body_chunked_buf->pos;
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ }
++
++ while(remains) {
++ size_t chunked_len = 0;
++ char *buf = uwsgi_chunked_read(wsgi_req, &chunked_len, timeout, 0);
++ if (chunked_len == 0) {
++ break;
++ }
++ if (uwsgi_buffer_append(wsgi_req->body_chunked_buf, buf, chunked_len)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++
++ if (chunked_len > remains) {
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos - (chunked_len - remains))) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos - (chunked_len - remains));
++ return ret;
++ }
++ remains -= chunked_len;
++ }
++
++ if (uwsgi_buffer_append(ret, wsgi_req->body_chunked_buf->buf, wsgi_req->body_chunked_buf->pos)) {
++ uwsgi_buffer_destroy(ret);
++ return NULL;
++ }
++ uwsgi_buffer_decapitate(wsgi_req->body_chunked_buf, wsgi_req->body_chunked_buf->pos);
++ return ret;
++}
++
+ char *uwsgi_chunked_read(struct wsgi_request *wsgi_req, size_t *len, int timeout, int nb) {
+
+ if (!wsgi_req->chunked_input_buf) {
+Index: uwsgi/core/protocol.c
+===================================================================
+--- uwsgi.orig/core/protocol.c
++++ uwsgi/core/protocol.c
+@@ -560,6 +560,12 @@ static int uwsgi_proto_check_22(struct w
+ wsgi_req->scheme_len = len;
+ }
+
++ if (!uwsgi_proto_key("HTTP_TRANSFER_ENCODING", 22)) {
++ if (!uwsgi_strnicmp(buf, len, "chunked", 7)) {
++ wsgi_req->body_is_chunked = 1;
++ }
++ }
++
+ return 0;
+ }
+
+Index: uwsgi/core/utils.c
+===================================================================
+--- uwsgi.orig/core/utils.c
++++ uwsgi/core/utils.c
+@@ -1182,6 +1182,10 @@ void uwsgi_close_request(struct wsgi_req
+ uwsgi_buffer_destroy(wsgi_req->chunked_input_buf);
+ }
+
++ if (wsgi_req->body_chunked_buf) {
++ uwsgi_buffer_destroy(wsgi_req->body_chunked_buf);
++ }
++
+ // free websocket engine
+ if (wsgi_req->websocket_buf) {
+ uwsgi_buffer_destroy(wsgi_req->websocket_buf);
+Index: uwsgi/plugins/python/python_plugin.c
+===================================================================
+--- uwsgi.orig/plugins/python/python_plugin.c
++++ uwsgi/plugins/python/python_plugin.c
+@@ -174,6 +174,8 @@ struct uwsgi_option uwsgi_python_options
+
+ {"python-worker-override", required_argument, 0, "override worker with the specified python script", uwsgi_opt_set_str, &up.worker_override, 0},
+
++ {"wsgi-manage-chunked-input", no_argument, 0, "manage chunked input via the wsgi.input_terminated extension", uwsgi_opt_true, &up.wsgi_manage_chunked_input, 0},
++
+ {0, 0, 0, 0, 0, 0, 0},
+ };
+
+Index: uwsgi/plugins/python/uwsgi_python.h
+===================================================================
+--- uwsgi.orig/plugins/python/uwsgi_python.h
++++ uwsgi/plugins/python/uwsgi_python.h
+@@ -214,6 +214,8 @@ struct uwsgi_python {
+ int wsgi_disable_file_wrapper;
+
+ char *worker_override;
++
++ int wsgi_manage_chunked_input;
+ };
+
+
+Index: uwsgi/plugins/python/wsgi_handlers.c
+===================================================================
+--- uwsgi.orig/plugins/python/wsgi_handlers.c
++++ uwsgi/plugins/python/wsgi_handlers.c
+@@ -62,7 +62,20 @@ static PyObject *uwsgi_Input_read(uwsgi_
+ ssize_t rlen = 0;
+
+ UWSGI_RELEASE_GIL
+- char *buf = uwsgi_request_body_read(wsgi_req, arg_len, &rlen);
++ char *buf = NULL;
++ if (wsgi_req->body_is_chunked && up.wsgi_manage_chunked_input) {
++ struct uwsgi_buffer *ubuf = uwsgi_chunked_read_smart(wsgi_req, arg_len, uwsgi.socket_timeout);
++ UWSGI_GET_GIL
++ if (!ubuf) {
++ return PyErr_Format(PyExc_IOError, "error during chunked read(%ld) on wsgi.input", arg_len);
++ }
++ PyObject *ret = PyString_FromStringAndSize(ubuf->buf, ubuf->pos);
++ uwsgi_buffer_destroy(ubuf);
++ return ret;
++ }
++ else {
++ buf = uwsgi_request_body_read(wsgi_req, arg_len, &rlen);
++ }
+ UWSGI_GET_GIL
+ if (buf == uwsgi.empty) {
+ return PyString_FromString("");
+Index: uwsgi/plugins/python/wsgi_subhandler.c
+===================================================================
+--- uwsgi.orig/plugins/python/wsgi_subhandler.c
++++ uwsgi/plugins/python/wsgi_subhandler.c
+@@ -179,6 +179,15 @@ void *uwsgi_request_subhandler_wsgi(stru
+
+ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input", wsgi_req->async_input);
+
++ if (up.wsgi_manage_chunked_input) {
++ if (wsgi_req->body_is_chunked) {
++ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input_terminated", Py_True);
++ }
++ else {
++ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.input_terminated", Py_False);
++ }
++ }
++
+ if (!up.wsgi_disable_file_wrapper)
+ PyDict_SetItemString(wsgi_req->async_environ, "wsgi.file_wrapper", wi->sendfile);
+
+Index: uwsgi/t/python/wsgi_chunked.py
+===================================================================
+--- /dev/null
++++ uwsgi/t/python/wsgi_chunked.py
+@@ -0,0 +1,16 @@
++def application(environ, start_response):
++ print(environ)
++ start_response('200 OK', [])
++ if not environ['wsgi.input_terminated']:
++ return []
++# print(environ['wsgi.input'].read())
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(2)
++ print(data)
++ data = environ['wsgi.input'].read(6)
++ print(data)
++ print(environ['wsgi.input'].read())
++ return [data]
+Index: uwsgi/uwsgi.h
+===================================================================
+--- uwsgi.orig/uwsgi.h
++++ uwsgi/uwsgi.h
+@@ -1617,6 +1617,9 @@ struct wsgi_request {
+ // used for protocol parsers requiring EOF signaling
+ int proto_parser_eof;
+
++ int body_is_chunked;
++ struct uwsgi_buffer *body_chunked_buf;
++
+ // 64bit range, deprecates size_t __range_from, __range_to
+ enum uwsgi_range range_parsed;
+ int64_t range_from;
+@@ -4506,6 +4509,7 @@ struct uwsgi_buffer *uwsgi_websocket_rec
+ struct uwsgi_buffer *uwsgi_websocket_recv_nb(struct wsgi_request *);
+
+ char *uwsgi_chunked_read(struct wsgi_request *, size_t *, int, int);
++struct uwsgi_buffer *uwsgi_chunked_read_smart(struct wsgi_request *, size_t, int);
+
+ uint16_t uwsgi_be16(char *);
+ uint32_t uwsgi_be32(char *);
diff --git a/debian/patches/series b/debian/patches/series
index b8c2938a..82ecc1f0 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -20,3 +20,4 @@
1016_readline.patch
1017_python3-compat.patch
2001_ensure_verbose_build.patch
+Add_support_for_chunked_encoding.patch
--
2.30.2
More information about the pkg-uWSGI-devel
mailing list