From 82b37ea0240c0e77857149d80beb4dda2b095dbb Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Tue, 30 Jan 2024 14:06:53 +0100 Subject: [PATCH 1/3] http: support basic access authentication --- lib/std/http/Client.zig | 60 ++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index 8438a7bbd3..0e50e25dd3 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -657,7 +657,7 @@ pub const Request = struct { }; } - pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; + pub const SendError = Allocator.Error || Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; pub const SendOptions = struct { /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped. @@ -695,6 +695,14 @@ pub const Request = struct { try w.writeAll("\r\n"); } + if ((req.uri.user != null or req.uri.password != null) and + !req.headers.contains("authorization")) + { + try w.writeAll("Authorization: "); + try w.writeAll(try basicAuthorizationValue(req.arena.allocator(), req.uri)); + try w.writeAll("\r\n"); + } + if (!req.headers.contains("user-agent")) { try w.writeAll("User-Agent: zig/"); try w.writeAll(builtin.zig_version_string); @@ -1122,19 +1130,11 @@ pub fn loadDefaultProxies(client: *Client) !void { }, }; - if (uri.user != null and uri.password != null) { - const prefix = "Basic "; + if (uri.user != null or uri.password != null) { + const authorization = try basicAuthorizationValue(client.allocator, uri); + defer client.allocator.free(authorization); - const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); - defer client.allocator.free(unencoded); - - const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); - defer client.allocator.free(buffer); - - const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); - @memcpy(buffer[0..prefix.len], prefix); - - try client.http_proxy.?.headers.append("proxy-authorization", result); + try client.http_proxy.?.headers.append("proxy-authorization", authorization); } } @@ -1173,23 +1173,33 @@ pub fn loadDefaultProxies(client: *Client) !void { }, }; - if (uri.user != null and uri.password != null) { - const prefix = "Basic "; + if (uri.user != null or uri.password != null) { + const authorization = try basicAuthorizationValue(client.allocator, uri); + defer client.allocator.free(authorization); - const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); - defer client.allocator.free(unencoded); - - const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); - defer client.allocator.free(buffer); - - const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); - @memcpy(buffer[0..prefix.len], prefix); - - try client.https_proxy.?.headers.append("proxy-authorization", result); + try client.https_proxy.?.headers.append("proxy-authorization", authorization); } } } +pub fn basicAuthorizationValue( + allocator: Allocator, + uri: Uri, +) Allocator.Error![]const u8 { + const prefix = "Basic "; + + const unencoded = try std.fmt.allocPrint(allocator, "{s}:{s}", .{ uri.user orelse "", uri.password orelse "" }); + defer allocator.free(unencoded); + + const buffer = try allocator.alloc(u8, prefix.len + std.base64.standard.Encoder.calcSize(unencoded.len)); + errdefer allocator.free(buffer); + + @memcpy(buffer[0..prefix.len], prefix); + _ = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); + + return buffer; +} + pub const ConnectTcpError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed }; /// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open. From a111f805cd6cc82952786d0ffccb5a31c68f6353 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Wed, 31 Jan 2024 14:44:34 +0100 Subject: [PATCH 2/3] http: avoid allocator use when encoding basic authorization --- lib/std/http/Client.zig | 68 +++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index 0e50e25dd3..c015569609 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -339,7 +339,7 @@ pub const Connection = struct { /// Writes the given buffer to the connection. pub fn write(conn: *Connection, buffer: []const u8) WriteError!usize { - if (conn.write_end + buffer.len > conn.write_buf.len) { + if (conn.write_buf.len - conn.write_end < buffer.len) { try conn.flush(); if (buffer.len > conn.write_buf.len) { @@ -354,6 +354,13 @@ pub const Connection = struct { return buffer.len; } + /// Returns a buffer to be filled with exactly len bytes to write to the connection. + pub fn allocWriteBuffer(conn: *Connection, len: BufferSize) WriteError![]u8 { + if (conn.write_buf.len - conn.write_end < len) try conn.flush(); + defer conn.write_end += len; + return conn.write_buf[conn.write_end..][0..len]; + } + /// Flushes the write buffer to the connection. pub fn flush(conn: *Connection) WriteError!void { if (conn.write_end == 0) return; @@ -657,7 +664,7 @@ pub const Request = struct { }; } - pub const SendError = Allocator.Error || Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; + pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; pub const SendOptions = struct { /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped. @@ -699,7 +706,10 @@ pub const Request = struct { !req.headers.contains("authorization")) { try w.writeAll("Authorization: "); - try w.writeAll(try basicAuthorizationValue(req.arena.allocator(), req.uri)); + const authorization = try req.connection.?.allocWriteBuffer( + @intCast(basic_authorization.valueLengthFromUri(req.uri)), + ); + std.debug.assert(basic_authorization.value(req.uri, authorization).len == authorization.len); try w.writeAll("\r\n"); } @@ -1131,10 +1141,8 @@ pub fn loadDefaultProxies(client: *Client) !void { }; if (uri.user != null or uri.password != null) { - const authorization = try basicAuthorizationValue(client.allocator, uri); - defer client.allocator.free(authorization); - - try client.http_proxy.?.headers.append("proxy-authorization", authorization); + var authorization: [basic_authorization.max_value_len]u8 = undefined; + try client.http_proxy.?.headers.append("proxy-authorization", basic_authorization.value(uri, &authorization)); } } @@ -1174,31 +1182,45 @@ pub fn loadDefaultProxies(client: *Client) !void { }; if (uri.user != null or uri.password != null) { - const authorization = try basicAuthorizationValue(client.allocator, uri); - defer client.allocator.free(authorization); - - try client.https_proxy.?.headers.append("proxy-authorization", authorization); + var authorization: [basic_authorization.max_value_len]u8 = undefined; + try client.https_proxy.?.headers.append("proxy-authorization", basic_authorization.value(uri, &authorization)); } } } -pub fn basicAuthorizationValue( - allocator: Allocator, - uri: Uri, -) Allocator.Error![]const u8 { +pub const basic_authorization = struct { + pub const max_user_len = 255; + pub const max_password_len = 255; + pub const max_value_len = valueLength(max_user_len, max_password_len); + const prefix = "Basic "; - const unencoded = try std.fmt.allocPrint(allocator, "{s}:{s}", .{ uri.user orelse "", uri.password orelse "" }); - defer allocator.free(unencoded); + pub fn valueLength(user_len: usize, password_len: usize) usize { + return prefix.len + std.base64.standard.Encoder.calcSize(user_len + 1 + password_len); + } - const buffer = try allocator.alloc(u8, prefix.len + std.base64.standard.Encoder.calcSize(unencoded.len)); - errdefer allocator.free(buffer); + pub fn valueLengthFromUri(uri: Uri) usize { + return valueLength( + if (uri.user) |user| user.len else 0, + if (uri.password) |password| password.len else 0, + ); + } - @memcpy(buffer[0..prefix.len], prefix); - _ = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); + pub fn value(uri: Uri, out: []u8) []u8 { + std.debug.assert(uri.user == null or uri.user.?.len <= max_user_len); + std.debug.assert(uri.password == null or uri.password.?.len <= max_password_len); - return buffer; -} + @memcpy(out[0..prefix.len], prefix); + + var buf: [max_user_len + ":".len + max_password_len]u8 = undefined; + const unencoded = std.fmt.bufPrint(&buf, "{s}:{s}", .{ + uri.user orelse "", uri.password orelse "", + }) catch unreachable; + const base64 = std.base64.standard.Encoder.encode(out[prefix.len..], unencoded); + + return out[0 .. prefix.len + base64.len]; + } +}; pub const ConnectTcpError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed }; From c1e7d0c08f96f390a641b082b33a8a8717cdd706 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Wed, 31 Jan 2024 14:33:17 +0100 Subject: [PATCH 3/3] http: optimize allocations for proxy basic authorization --- lib/std/http/Client.zig | 12 ++++++---- lib/std/http/Headers.zig | 52 +++++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index c015569609..2c45b36173 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -1141,8 +1141,10 @@ pub fn loadDefaultProxies(client: *Client) !void { }; if (uri.user != null or uri.password != null) { - var authorization: [basic_authorization.max_value_len]u8 = undefined; - try client.http_proxy.?.headers.append("proxy-authorization", basic_authorization.value(uri, &authorization)); + const authorization = try client.allocator.alloc(u8, basic_authorization.valueLengthFromUri(uri)); + errdefer client.allocator.free(authorization); + std.debug.assert(basic_authorization.value(uri, authorization).len == authorization.len); + try client.http_proxy.?.headers.appendOwned(.{ .unowned = "proxy-authorization" }, .{ .owned = authorization }); } } @@ -1182,8 +1184,10 @@ pub fn loadDefaultProxies(client: *Client) !void { }; if (uri.user != null or uri.password != null) { - var authorization: [basic_authorization.max_value_len]u8 = undefined; - try client.https_proxy.?.headers.append("proxy-authorization", basic_authorization.value(uri, &authorization)); + const authorization = try client.allocator.alloc(u8, basic_authorization.valueLengthFromUri(uri)); + errdefer client.allocator.free(authorization); + std.debug.assert(basic_authorization.value(uri, authorization).len == authorization.len); + try client.https_proxy.?.headers.appendOwned(.{ .unowned = "proxy-authorization" }, .{ .owned = authorization }); } } } diff --git a/lib/std/http/Headers.zig b/lib/std/http/Headers.zig index c35775e65b..653ec05126 100644 --- a/lib/std/http/Headers.zig +++ b/lib/std/http/Headers.zig @@ -91,30 +91,64 @@ pub const Headers = struct { /// /// If the `owned` field is true, both name and value will be copied. pub fn append(headers: *Headers, name: []const u8, value: []const u8) !void { + try headers.appendOwned(.{ .unowned = name }, .{ .unowned = value }); + } + + pub const OwnedString = union(enum) { + /// A string allocated by the `allocator` field. + owned: []u8, + /// A string to be copied by the `allocator` field. + unowned: []const u8, + }; + + /// Appends a header to the list. + /// + /// If the `owned` field is true, `name` and `value` will be copied if unowned. + pub fn appendOwned(headers: *Headers, name: OwnedString, value: OwnedString) !void { const n = headers.list.items.len; + try headers.list.ensureUnusedCapacity(headers.allocator, 1); - const value_duped = if (headers.owned) try headers.allocator.dupe(u8, value) else value; - errdefer if (headers.owned) headers.allocator.free(value_duped); + const owned_value = switch (value) { + .owned => |owned| owned, + .unowned => |unowned| if (headers.owned) + try headers.allocator.dupe(u8, unowned) + else + unowned, + }; + errdefer if (value == .unowned and headers.owned) headers.allocator.free(owned_value); - var entry = Field{ .name = undefined, .value = value_duped }; + var entry = Field{ .name = undefined, .value = owned_value }; + + if (headers.index.getEntry(switch (name) { + inline else => |string| string, + })) |kv| { + defer switch (name) { + .owned => |owned| headers.allocator.free(owned), + .unowned => {}, + }; - if (headers.index.getEntry(name)) |kv| { entry.name = kv.key_ptr.*; try kv.value_ptr.append(headers.allocator, n); } else { - const name_duped = if (headers.owned) try std.ascii.allocLowerString(headers.allocator, name) else name; - errdefer if (headers.owned) headers.allocator.free(name_duped); + const owned_name = switch (name) { + .owned => |owned| owned, + .unowned => |unowned| if (headers.owned) + try std.ascii.allocLowerString(headers.allocator, unowned) + else + unowned, + }; + errdefer if (name == .unowned and headers.owned) headers.allocator.free(owned_name); - entry.name = name_duped; + entry.name = owned_name; var new_index = try HeaderIndexList.initCapacity(headers.allocator, 1); errdefer new_index.deinit(headers.allocator); new_index.appendAssumeCapacity(n); - try headers.index.put(headers.allocator, name_duped, new_index); + try headers.index.put(headers.allocator, owned_name, new_index); } - try headers.list.append(headers.allocator, entry); + headers.list.appendAssumeCapacity(entry); } /// Returns true if this list of headers contains the given name.