zig

blob 9e2ebd2d (17553B) - Raw

---

 const std = @import("std");
 const http = std.http;
 const Uri = std.Uri;
 const mem = std.mem;
 const assert = std.debug.assert;
 
 const Client = @import("../Client.zig");
 const Connection = Client.Connection;
 const ConnectionNode = Client.ConnectionPool.Node;
 const Response = @import("Response.zig");
 
 const Request = @This();
 
 const read_buffer_size = 8192;
 const ReadBufferIndex = std.math.IntFittingRange(0, read_buffer_size);
 
 uri: Uri,
 client: *Client,
 connection: *ConnectionNode,
 response: Response,
 /// These are stored in Request so that they are available when following
 /// redirects.
 headers: Headers,
 
 redirects_left: u32,
 handle_redirects: bool,
 compression_init: bool,
 
 /// Used as a allocator for resolving redirects locations.
 arena: std.heap.ArenaAllocator,
 
 /// Read buffer for the connection. This is used to pull in large amounts of data from the connection even if the user asks for a small amount. This can probably be removed with careful planning.
 read_buffer: [read_buffer_size]u8 = undefined,
 read_buffer_start: ReadBufferIndex = 0,
 read_buffer_len: ReadBufferIndex = 0,
 
 pub const RequestTransfer = union(enum) {
     content_length: u64,
     chunked: void,
     none: void,
 };
 
 pub const Headers = struct {
     version: http.Version = .@"HTTP/1.1",
     method: http.Method = .GET,
     user_agent: []const u8 = "zig (std.http)",
     connection: http.Connection = .keep_alive,
     transfer_encoding: RequestTransfer = .none,
 
     custom: []const http.CustomHeader = &[_]http.CustomHeader{},
 };
 
 pub const Options = struct {
     handle_redirects: bool = true,
     max_redirects: u32 = 3,
     header_strategy: HeaderStrategy = .{ .dynamic = 16 * 1024 },
 
     pub const HeaderStrategy = union(enum) {
         /// In this case, the client's Allocator will be used to store the
         /// entire HTTP header. This value is the maximum total size of
         /// HTTP headers allowed, otherwise
         /// error.HttpHeadersExceededSizeLimit is returned from read().
         dynamic: usize,
         /// This is used to store the entire HTTP header. If the HTTP
         /// header is too big to fit, `error.HttpHeadersExceededSizeLimit`
         /// is returned from read(). When this is used, `error.OutOfMemory`
         /// cannot be returned from `read()`.
         static: []u8,
     };
 };
 
 /// Frees all resources associated with the request.
 pub fn deinit(req: *Request) void {
     switch (req.response.compression) {
         .none => {},
         .deflate => |*deflate| deflate.deinit(),
         .gzip => |*gzip| gzip.deinit(),
         .zstd => |*zstd| zstd.deinit(),
     }
 
     if (req.response.header_bytes_owned) {
         req.response.header_bytes.deinit(req.client.allocator);
     }
 
     if (!req.response.done) {
         // If the response wasn't fully read, then we need to close the connection.
         req.connection.data.closing = true;
         req.client.connection_pool.release(req.client, req.connection);
     }
 
     req.arena.deinit();
     req.* = undefined;
 }
 
 pub const ReadRawError = Connection.ReadError || Uri.ParseError || Client.RequestError || error{
     UnexpectedEndOfStream,
     TooManyHttpRedirects,
     HttpRedirectMissingLocation,
     HttpHeadersInvalid,
 };
 
 pub const ReaderRaw = std.io.Reader(*Request, ReadRawError, readRaw);
 
 /// Read from the underlying stream, without decompressing or parsing the headers. Must be called
 /// after waitForCompleteHead() has returned successfully.
 pub fn readRaw(req: *Request, buffer: []u8) ReadRawError!usize {
     assert(req.response.state.isContent());
 
     var index: usize = 0;
     while (index == 0) {
         const amt = try req.readRawAdvanced(buffer[index..]);
         if (amt == 0 and req.response.done) break;
         index += amt;
     }
 
     return index;
 }
 
 fn checkForCompleteHead(req: *Request, buffer: []u8) !usize {
     switch (req.response.state) {
         .invalid => unreachable,
         .start, .seen_r, .seen_rn, .seen_rnr => {},
         else => return 0, // No more headers to read.
     }
 
     const i = req.response.findHeadersEnd(buffer[0..]);
     if (req.response.state == .invalid) return error.HttpHeadersInvalid;
 
     const headers_data = buffer[0..i];
     if (req.response.header_bytes.items.len + headers_data.len > req.response.max_header_bytes) {
         return error.HttpHeadersExceededSizeLimit;
     }
     try req.response.header_bytes.appendSlice(req.client.allocator, headers_data);
 
     if (req.response.state == .finished) {
         req.response.headers = try Response.Headers.parse(req.response.header_bytes.items);
 
         if (req.response.headers.upgrade) |_| {
             req.connection.data.closing = false;
             req.response.done = true;
             return i;
         }
 
         if (req.response.headers.connection == .keep_alive) {
             req.connection.data.closing = false;
         } else {
             req.connection.data.closing = true;
         }
 
         if (req.response.headers.transfer_encoding) |transfer_encoding| {
             switch (transfer_encoding) {
                 .chunked => {
                     req.response.next_chunk_length = 0;
                     req.response.state = .chunk_size;
                 },
             }
         } else if (req.response.headers.content_length) |content_length| {
             req.response.next_chunk_length = content_length;
 
             if (content_length == 0) req.response.done = true;
         } else {
             req.response.done = true;
         }
 
         return i;
     }
 
     return 0;
 }
 
 pub const WaitForCompleteHeadError = ReadRawError || error{
     UnexpectedEndOfStream,
 
     HttpHeadersExceededSizeLimit,
     ShortHttpStatusLine,
     BadHttpVersion,
     HttpHeaderContinuationsUnsupported,
     HttpTransferEncodingUnsupported,
     HttpConnectionHeaderUnsupported,
 };
 
 /// Reads a complete response head. Any leftover data is stored in the request. This function is idempotent.
 pub fn waitForCompleteHead(req: *Request) WaitForCompleteHeadError!void {
     if (req.response.state.isContent()) return;
 
     while (true) {
         const nread = try req.connection.data.read(req.read_buffer[0..]);
         const amt = try checkForCompleteHead(req, req.read_buffer[0..nread]);
 
         if (amt != 0) {
             req.read_buffer_start = @intCast(ReadBufferIndex, amt);
             req.read_buffer_len = @intCast(ReadBufferIndex, nread);
             return;
         } else if (nread == 0) {
             return error.UnexpectedEndOfStream;
         }
     }
 }
 
 /// This one can return 0 without meaning EOF.
 fn readRawAdvanced(req: *Request, buffer: []u8) !usize {
     assert(req.response.state.isContent());
     if (req.response.done) return 0;
 
     // var in: []const u8 = undefined;
     if (req.read_buffer_start == req.read_buffer_len) {
         const nread = try req.connection.data.read(req.read_buffer[0..]);
         if (nread == 0) return error.UnexpectedEndOfStream;
 
         req.read_buffer_start = 0;
         req.read_buffer_len = @intCast(ReadBufferIndex, nread);
     }
 
     var out_index: usize = 0;
     while (true) {
         switch (req.response.state) {
             .invalid, .start, .seen_r, .seen_rn, .seen_rnr => unreachable,
             .finished => {
                 // TODO https://github.com/ziglang/zig/issues/14039
                 const buf_avail = req.read_buffer_len - req.read_buffer_start;
                 const data_avail = req.response.next_chunk_length;
                 const out_avail = buffer.len;
 
                 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
                     const can_read = @intCast(usize, @min(buf_avail, data_avail));
                     req.response.next_chunk_length -= can_read;
 
                     if (req.response.next_chunk_length == 0) {
                         req.client.connection_pool.release(req.client, req.connection);
                         req.connection = undefined;
                         req.response.done = true;
                     }
 
                     return 0; // skip over as much data as possible
                 }
 
                 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
                 req.response.next_chunk_length -= can_read;
 
                 mem.copy(u8, buffer[0..], req.read_buffer[req.read_buffer_start..][0..can_read]);
                 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
 
                 if (req.response.next_chunk_length == 0) {
                     req.client.connection_pool.release(req.client, req.connection);
                     req.connection = undefined;
                     req.response.done = true;
                 }
 
                 return can_read;
             },
             .chunk_size_prefix_r => switch (req.read_buffer_len - req.read_buffer_start) {
                 0 => return out_index,
                 1 => switch (req.read_buffer[req.read_buffer_start]) {
                     '\r' => {
                         req.response.state = .chunk_size_prefix_n;
                         return out_index;
                     },
                     else => {
                         req.response.state = .invalid;
                         return error.HttpHeadersInvalid;
                     },
                 },
                 else => switch (int16(req.read_buffer[req.read_buffer_start..][0..2])) {
                     int16("\r\n") => {
                         req.read_buffer_start += 2;
                         req.response.state = .chunk_size;
                         continue;
                     },
                     else => {
                         req.response.state = .invalid;
                         return error.HttpHeadersInvalid;
                     },
                 },
             },
             .chunk_size_prefix_n => switch (req.read_buffer_len - req.read_buffer_start) {
                 0 => return out_index,
                 else => switch (req.read_buffer[req.read_buffer_start]) {
                     '\n' => {
                         req.read_buffer_start += 1;
                         req.response.state = .chunk_size;
                         continue;
                     },
                     else => {
                         req.response.state = .invalid;
                         return error.HttpHeadersInvalid;
                     },
                 },
             },
             .chunk_size, .chunk_r => {
                 const i = req.response.findChunkedLen(req.read_buffer[req.read_buffer_start..req.read_buffer_len]);
                 switch (req.response.state) {
                     .invalid => return error.HttpHeadersInvalid,
                     .chunk_data => {
                         if (req.response.next_chunk_length == 0) {
                             req.response.done = true;
                             req.client.connection_pool.release(req.client, req.connection);
                             req.connection = undefined;
 
                             return out_index;
                         }
 
                         req.read_buffer_start += @intCast(ReadBufferIndex, i);
                         continue;
                     },
                     .chunk_size => return out_index,
                     else => unreachable,
                 }
             },
             .chunk_data => {
                 // TODO https://github.com/ziglang/zig/issues/14039
                 const buf_avail = req.read_buffer_len - req.read_buffer_start;
                 const data_avail = req.response.next_chunk_length;
                 const out_avail = buffer.len - out_index;
 
                 if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
                     const can_read = @intCast(usize, @min(buf_avail, data_avail));
                     req.response.next_chunk_length -= can_read;
 
                     if (req.response.next_chunk_length == 0) {
                         req.client.connection_pool.release(req.client, req.connection);
                         req.connection = undefined;
                         req.response.done = true;
                         continue;
                     }
 
                     return 0; // skip over as much data as possible
                 }
 
                 const can_read = @intCast(usize, @min(@min(buf_avail, data_avail), out_avail));
                 req.response.next_chunk_length -= can_read;
 
                 mem.copy(u8, buffer[out_index..], req.read_buffer[req.read_buffer_start..][0..can_read]);
                 req.read_buffer_start += @intCast(ReadBufferIndex, can_read);
                 out_index += can_read;
 
                 if (req.response.next_chunk_length == 0) {
                     req.response.state = .chunk_size_prefix_r;
 
                     continue;
                 }
 
                 return out_index;
             },
         }
     }
 }
 
 pub const ReadError = Client.DeflateDecompressor.Error || Client.GzipDecompressor.Error || Client.ZstdDecompressor.Error || WaitForCompleteHeadError || error{ BadHeader, InvalidCompression, StreamTooLong, InvalidWindowSize, CompressionNotSupported };
 
 pub const Reader = std.io.Reader(*Request, ReadError, read);
 
 pub fn reader(req: *Request) Reader {
     return .{ .context = req };
 }
 
 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
     while (true) {
         if (!req.response.state.isContent()) try req.waitForCompleteHead();
 
         if (req.handle_redirects and req.response.headers.status.class() == .redirect) {
             assert(try req.readRaw(buffer) == 0);
 
             if (req.redirects_left == 0) return error.TooManyHttpRedirects;
 
             const location = req.response.headers.location orelse
                 return error.HttpRedirectMissingLocation;
             const new_url = Uri.parse(location) catch try Uri.parseWithoutScheme(location);
 
             var new_arena = std.heap.ArenaAllocator.init(req.client.allocator);
             const resolved_url = try req.uri.resolve(new_url, false, new_arena.allocator());
             errdefer new_arena.deinit();
 
             req.arena.deinit();
             req.arena = new_arena;
 
             const new_req = try req.client.request(resolved_url, req.headers, .{
                 .max_redirects = req.redirects_left - 1,
                 .header_strategy = if (req.response.header_bytes_owned) .{
                     .dynamic = req.response.max_header_bytes,
                 } else .{
                     .static = req.response.header_bytes.unusedCapacitySlice(),
                 },
             });
             req.deinit();
             req.* = new_req;
         } else {
             break;
         }
     }
 
     if (req.response.compression == .none) {
         if (req.response.headers.transfer_compression) |compression| {
             switch (compression) {
                 .compress => return error.CompressionNotSupported,
                 .deflate => req.response.compression = .{
                     .deflate = try std.compress.zlib.zlibStream(req.client.allocator, ReaderRaw{ .context = req }),
                 },
                 .gzip => req.response.compression = .{
                     .gzip = try std.compress.gzip.decompress(req.client.allocator, ReaderRaw{ .context = req }),
                 },
                 .zstd => req.response.compression = .{
                     .zstd = std.compress.zstd.decompressStream(req.client.allocator, ReaderRaw{ .context = req }),
                 },
             }
         }
     }
 
     return switch (req.response.compression) {
         .deflate => |*deflate| try deflate.read(buffer),
         .gzip => |*gzip| try gzip.read(buffer),
         .zstd => |*zstd| try zstd.read(buffer),
         else => try req.readRaw(buffer),
     };
 }
 
 pub fn readAll(req: *Request, buffer: []u8) !usize {
     var index: usize = 0;
     while (index < buffer.len) {
         const amt = try read(req, buffer[index..]);
         if (amt == 0) break;
         index += amt;
     }
     return index;
 }
 
 pub const WriteError = Connection.WriteError || error{MessageTooLong};
 
 pub const Writer = std.io.Writer(*Request, WriteError, write);
 
 pub fn writer(req: *Request) Writer {
     return .{ .context = req };
 }
 
 /// Write `bytes` to the server. The `transfer_encoding` request header determines how data will be sent.
 pub fn write(req: *Request, bytes: []const u8) !usize {
     switch (req.headers.transfer_encoding) {
         .chunked => {
             try req.connection.data.writer().print("{x}\r\n", .{bytes.len});
             try req.connection.data.writeAll(bytes);
             try req.connection.data.writeAll("\r\n");
 
             return bytes.len;
         },
         .content_length => |*len| {
             if (len.* < bytes.len) return error.MessageTooLong;
 
             const amt = try req.connection.data.write(bytes);
             len.* -= amt;
             return amt;
         },
         .none => return error.NotWriteable,
     }
 }
 
 /// Finish the body of a request. This notifies the server that you have no more data to send.
 pub fn finish(req: *Request) !void {
     switch (req.headers.transfer_encoding) {
         .chunked => try req.connection.data.writeAll("0\r\n"),
         .content_length => |len| if (len != 0) return error.MessageNotCompleted,
         .none => {},
     }
 }
 
 inline fn int16(array: *const [2]u8) u16 {
     return @bitCast(u16, array.*);
 }
 
 inline fn int32(array: *const [4]u8) u32 {
     return @bitCast(u32, array.*);
 }
 
 inline fn int64(array: *const [8]u8) u64 {
     return @bitCast(u64, array.*);
 }
 
 test {
     const builtin = @import("builtin");
 
     if (builtin.os.tag == .wasi) return error.SkipZigTest;
 
     _ = Response;
 }