Allow body-up-to-EOF HTTP responses

Implements the same heuristic as Curl (and web browsers): if no `Content-Length`, no `Connection: keep-alive` and no chunked transfer encoding, assume th rest of the data until EOF is the body, gracefully setting the HTTP client back to the disconnected state.

Theoretically, this is not compliant with HTTP 1.1, by which `keep-alive` is the default, but in practice, an explicit header is sent by servers.

4 files changed, 52 insertions, 15 deletions

diff --git a/core/io/http_client.cpp b/core/io/http_client.cpp
index 9e301ccac..8d85e7822 100644
--- a/core/io/http_client.cpp
+++ b/core/io/http_client.cpp
@@ -248,6 +248,7 @@ void HTTPClient::close() {
 	body_size = 0;
 	body_left = 0;
 	chunk_left = 0;
+	read_until_eof = false;
 	response_num = 0;
 }
 
@@ -352,10 +353,17 @@ Error HTTPClient::poll() {
 					chunked = false;
 					body_left = 0;
 					chunk_left = 0;
+					read_until_eof = false;
 					response_str.clear();
 					response_headers.clear();
 					response_num = RESPONSE_OK;
 
+					// Per the HTTP 1.1 spec, keep-alive is the default, but in practice
+					// it's safe to assume it only if the explicit header is found, allowing
+					// to handle body-up-to-EOF responses on naive servers; that's what Curl
+					// and browsers do
+					bool keep_alive = false;
+
 					for (int i = 0; i < responses.size(); i++) {
 
 						String header = responses[i].strip_edges();
@@ -365,13 +373,14 @@ Error HTTPClient::poll() {
 						if (s.begins_with("content-length:")) {
 							body_size = s.substr(s.find(":") + 1, s.length()).strip_edges().to_int();
 							body_left = body_size;
-						}
 
-						if (s.begins_with("transfer-encoding:")) {
+						} else if (s.begins_with("transfer-encoding:")) {
 							String encoding = header.substr(header.find(":") + 1, header.length()).strip_edges();
 							if (encoding == "chunked") {
 								chunked = true;
 							}
+						} else if (s.begins_with("connection: keep-alive")) {
+							keep_alive = true;
 						}
 
 						if (i == 0 && responses[i].begins_with("HTTP")) {
@@ -384,11 +393,16 @@ Error HTTPClient::poll() {
 						}
 					}
 
-					if (body_size == 0 && !chunked) {
+					if (body_size || chunked) {
 
-						status = STATUS_CONNECTED; // Ready for new requests
-					} else {
 						status = STATUS_BODY;
+					} else if (!keep_alive) {
+
+						read_until_eof = true;
+						status = STATUS_BODY;
+					} else {
+
+						status = STATUS_CONNECTED;
 					}
 					return OK;
 				}
@@ -515,34 +529,53 @@ PoolByteArray HTTPClient::read_response_body_chunk() {
 
 	} else {
 
-		int to_read = MIN(body_left, read_chunk_size);
+		int to_read = !read_until_eof ? MIN(body_left, read_chunk_size) : read_chunk_size;
 		PoolByteArray ret;
 		ret.resize(to_read);
 		int _offset = 0;
-		while (to_read > 0) {
+		while (read_until_eof || to_read > 0) {
 			int rec = 0;
 			{
 				PoolByteArray::Write w = ret.write();
 				err = _get_http_data(w.ptr() + _offset, to_read, rec);
 			}
-			if (rec > 0) {
-				body_left -= rec;
-				to_read -= rec;
-				_offset += rec;
-			} else {
+			if (rec < 0) {
 				if (to_read > 0) // Ended up reading less
 					ret.resize(_offset);
 				break;
+			} else {
+				_offset += rec;
+				if (!read_until_eof) {
+					body_left -= rec;
+					to_read -= rec;
+				} else {
+					if (rec < to_read) {
+						ret.resize(_offset);
+						err = ERR_FILE_EOF;
+						break;
+					}
+					ret.resize(_offset + to_read);
+				}
 			}
 		}
-		if (body_left == 0) {
-			status = STATUS_CONNECTED;
+		if (!read_until_eof) {
+			if (body_left == 0) {
+				status = STATUS_CONNECTED;
+			}
+			return ret;
+		} else {
+			if (err == ERR_FILE_EOF) {
+				err = OK; // EOF is expected here
+				close();
+				return ret;
+			}
 		}
-		return ret;
 	}
 
 	if (err != OK) {
+
 		close();
+
 		if (err == ERR_FILE_EOF) {
 
 			status = STATUS_DISCONNECTED; // Server disconnected
@@ -602,6 +635,7 @@ HTTPClient::HTTPClient() {
 	body_size = 0;
 	chunked = false;
 	body_left = 0;
+	read_until_eof = false;
 	chunk_left = 0;
 	response_num = 0;
 	ssl = false;
diff --git a/core/io/http_client.h b/core/io/http_client.h
index 839012e70..38ec82ce8 100644
--- a/core/io/http_client.h
+++ b/core/io/http_client.h
@@ -173,6 +173,7 @@ private:
 	int chunk_left;
 	int body_size;
 	int body_left;
+	bool read_until_eof;
 
 	Ref<StreamPeerTCP> tcp_connection;
 	Ref<StreamPeer> connection;
diff --git a/drivers/unix/stream_peer_tcp_posix.cpp b/drivers/unix/stream_peer_tcp_posix.cpp
index 17112e5ab..b19bdc460 100644
--- a/drivers/unix/stream_peer_tcp_posix.cpp
+++ b/drivers/unix/stream_peer_tcp_posix.cpp
@@ -290,6 +290,7 @@ Error StreamPeerTCPPosix::read(uint8_t *p_buffer, int p_bytes, int &r_received,
 			status = STATUS_NONE;
 			peer_port = 0;
 			peer_host = IP_Address();
+			r_received = total_read;
 			return ERR_FILE_EOF;
 
 		} else {
diff --git a/drivers/windows/stream_peer_tcp_winsock.cpp b/drivers/windows/stream_peer_tcp_winsock.cpp
index 55775fc23..2faf3f377 100644
--- a/drivers/windows/stream_peer_tcp_winsock.cpp
+++ b/drivers/windows/stream_peer_tcp_winsock.cpp
@@ -212,6 +212,7 @@ Error StreamPeerTCPWinsock::read(uint8_t *p_buffer, int p_bytes, int &r_received
 			_block(sockfd, true, false);
 		} else if (read == 0) {
 			disconnect_from_host();
+			r_received = total_read;
 			return ERR_FILE_EOF;
 		} else {