requests: HTTP/1.1 with Content-Length and streaming raw

python-ecosys/requests/README.md

@@ -4,7 +4,8 @@ This module provides a lightweight version of the Python
 [requests](https://requests.readthedocs.io/en/latest/) library.
 
 It includes support for all HTTP verbs, https, json decoding of responses,
-redirects, basic authentication.
+redirects, basic authentication, HTTP/1.1 requests, and reading response
+bodies with Content-Length via streaming ``.raw`` or lazy ``.content``.
 
 ### Limitations
 
@@ -14,3 +15,10 @@ redirects, basic authentication.
 * Compressed requests/responses are not currently supported.
 * File upload is not supported.
 * Chunked encoding in responses is not supported.
+* HTTP keep-alive connection reuse is not supported (Connection: close by default).
+
+### Follow-up work
+
+* Chunked response bodies (separate PR).
+* TLS certificate verification (see micropython-lib issue #838).
+* ``stream=True`` incremental body reads (see issue #777).

python-ecosys/requests/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.10.2", pypi="requests")
+metadata(version="0.10.3", pypi="requests")
 
 package("requests")

python-ecosys/requests/requests/__init__.py

@@ -1,6 +1,26 @@
 import socket
 
 
+class BodyStream:
+    def __init__(self, sock, remaining):
+        self._sock = sock
+        self._remaining = remaining
+
+    def read(self, n=-1):
+        if self._remaining == 0:
+            return b""
+        if n < 0 or n > self._remaining:
+            n = self._remaining
+        data = self._sock.read(n)
+        self._remaining -= len(data)
+        if not data:
+            raise ValueError("Connection closed before Content-Length satisfied")
+        return data
+
+    def close(self):
+        self._sock.close()
+
+
 class Response:
     def __init__(self, f):
         self.raw = f
@@ -98,7 +118,7 @@ def request(
             context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
             context.verify_mode = tls.CERT_NONE
             s = context.wrap_socket(s, server_hostname=host)
-        s.write(b"%s /%s HTTP/1.0\r\n" % (method, path))
+        s.write(b"%s /%s HTTP/1.1\r\n" % (method, path))
 
         if "Host" not in headers:
             headers["Host"] = host
@@ -155,6 +175,7 @@ def request(
         reason = ""
         if len(l) > 2:
             reason = l[2].rstrip()
+        remaining = None
         while True:
             l = s.readline()
             if not l or l == b"\r\n":
@@ -173,7 +194,10 @@ def request(
             elif parse_headers is True:
                 l = str(l, "utf-8")
                 k, v = l.split(":", 1)
-                resp_d[k] = v.strip()
+                v = v.strip()
+                resp_d[k] = v
+                if k.lower() == "content-length":
+                    remaining = int(v)
             else:
                 parse_headers(l, resp_d)
     except OSError:
@@ -189,7 +213,10 @@ def request(
         else:
             return request(method, redirect, data, json, headers, stream)
     else:
-        resp = Response(s)
+        if remaining is not None:
+            resp = Response(BodyStream(s, remaining))
+        else:
+            resp = Response(s)
         resp.status_code = status
         resp.reason = reason
         if resp_d is not None: