motiejus/zig
1
Fork 0
Code Releases Activity

std: implement a cross platform file locking abstraction

Browse Source 
This modifies the lock semantics from using AccessMode to using
NtLockFile/NtUnlockFile.

This is a breaking change.
This commit is contained in:
Andrew Kelley
2021-06-28 16:46:37 -07:00
parent 488f68069b
commit 06129d7e3d
6 changed files with 239 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/langref.html.in
View File

@@ -10713,6 +10713,14 @@ fn readU32Be() u32 {}
See the Zig Standard Library for more examples.
</p>
{#header_close#}
{#header_open|Doc Comment Guidance#}
<ul>
<li>Omit any information that is redundant based on the name of the thing being documented.</li>
<li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
<li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
<li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
</ul>
{#header_close#}
{#header_close#}
{#header_open|Source Encoding#}
<p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>

64
lib/std/fs.zig
View File

@@ -883,24 +883,39 @@ pub const Dir = struct {
/// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
const w = os.windows;
return @as(File, .{
.handle = try os.windows.OpenFile(sub_path_w, .{
const file: File = .{
.handle = try w.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE |
(if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
(if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
.share_access = switch (flags.lock) {
.None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Exclusive => w.FILE_SHARE_DELETE,
},
.share_access_nonblocking = flags.lock_nonblocking,
.creation = w.FILE_OPEN,
.io_mode = flags.intended_io_mode,
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
});
};
var io: w.IO_STATUS_BLOCK = undefined;
const range_off: w.LARGE_INTEGER = 0;
const range_len: w.LARGE_INTEGER = 1;
const exclusive = switch (flags.lock) {
.None => return file,
.Shared => false,
.Exclusive => true,
};
try w.LockFile(
file.handle,
null,
null,
null,
&io,
&range_off,
&range_len,
null,
@boolToInt(flags.lock_nonblocking),
@boolToInt(exclusive),
);
return file;
}
/// Creates, opens, or overwrites a file with write access.
@@ -1019,16 +1034,10 @@ pub const Dir = struct {
pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
const w = os.windows;
const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
return @as(File, .{
const file: File = .{
.handle = try os.windows.OpenFile(sub_path_w, .{
.dir = self.fd,
.access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
.share_access = switch (flags.lock) {
.None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
.Exclusive => w.FILE_SHARE_DELETE,
},
.share_access_nonblocking = flags.lock_nonblocking,
.creation = if (flags.exclusive)
@as(u32, w.FILE_CREATE)
else if (flags.truncate)
@@ -1039,7 +1048,28 @@ pub const Dir = struct {
}),
.capable_io_mode = std.io.default_mode,
.intended_io_mode = flags.intended_io_mode,
});
};
var io: w.IO_STATUS_BLOCK = undefined;
const range_off: w.LARGE_INTEGER = 0;
const range_len: w.LARGE_INTEGER = 1;
const exclusive = switch (flags.lock) {
.None => return file,
.Shared => false,
.Exclusive => true,
};
try w.LockFile(
file.handle,
null,
null,
null,
&io,
&range_off,
&range_len,
null,
@boolToInt(flags.lock_nonblocking),
@boolToInt(exclusive),
);
return file;
}
pub const openRead = @compileError("deprecated in favor of openFile");

166
lib/std/fs/file.zig
View File

@@ -830,29 +830,25 @@ pub const File = struct {
return .{ .context = file };
}
pub const SetLockError = os.FlockError;
const range_off: windows.LARGE_INTEGER = 0;
const range_len: windows.LARGE_INTEGER = 1;
pub const LockError = error{
SystemResources,
} || os.UnexpectedError;
/// Blocks when an incompatible lock is held by another process.
/// `non_blocking` may be used to make a non-blocking request,
/// causing this function to possibly return `error.WouldBlock`.
/// A process may hold only one type of lock (shared or exclusive) on
/// a file. When a process terminates in any way, the lock is released.
///
/// Assumes the file is unlocked.
///
/// TODO: integrate with async I/O
pub fn setLock(file: File, lock: Lock, non_blocking: bool) SetLockError!void {
pub fn lock(file: File, l: Lock) LockError!void {
if (is_windows) {
const range_off: windows.LARGE_INTEGER = 0;
const range_len: windows.LARGE_INTEGER = 1;
const exclusive = switch (lock) {
.None => return windows.UnlockFile(
file.handle,
null,
&range_off,
&range_len,
null,
) catch |err| switch (err) {
error.RangeNotLocked => return,
else => |e| return e,
},
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
const exclusive = switch (l) {
.None => return,
.Shared => false,
.Exclusive => true,
};
@@ -861,19 +857,137 @@ pub const File = struct {
null,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
@boolToInt(non_blocking),
windows.FALSE, // non-blocking=false
@boolToInt(exclusive),
);
) catch |err| switch (err) {
error.WouldBlock => unreachable, // non-blocking=false
else => |e| return e,
};
} else {
return os.flock(file.handle, switch (l) {
.None => os.LOCK_UN,
.Shared => os.LOCK_SH,
.Exclusive => os.LOCK_EX,
}) catch |err| switch (err) {
error.WouldBlock => unreachable, // non-blocking=false
else => |e| return e,
};
}
}
/// Assumes the file is locked.
pub fn unlock(file: File) void {
if (is_windows) {
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
return windows.UnlockFile(
file.handle,
&io_status_block,
&range_off,
&range_len,
null,
) catch |err| switch (err) {
error.RangeNotLocked => unreachable, // Function assumes unlocked.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
} else {
return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
error.WouldBlock => unreachable, // unlocking can't block
error.SystemResources => unreachable, // We are deallocating resources.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
}
}
/// Attempts to obtain a lock, returning `true` if the lock is
/// obtained, and `false` if there was an existing incompatible lock held.
/// A process may hold only one type of lock (shared or exclusive) on
/// a file. When a process terminates in any way, the lock is released.
///
/// Assumes the file is unlocked.
///
/// TODO: integrate with async I/O
pub fn tryLock(file: File, l: Lock) LockError!bool {
if (is_windows) {
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
const exclusive = switch (l) {
.None => return,
.Shared => false,
.Exclusive => true,
};
windows.LockFile(
file.handle,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
windows.TRUE, // non-blocking=true
@boolToInt(exclusive),
) catch |err| switch (err) {
error.WouldBlock => return false,
else => |e| return e,
};
} else {
os.flock(file.handle, switch (l) {
.None => os.LOCK_UN,
.Shared => os.LOCK_SH | os.LOCK_NB,
.Exclusive => os.LOCK_EX | os.LOCK_NB,
}) catch |err| switch (err) {
error.WouldBlock => return false,
else => |e| return e,
};
}
return true;
}
/// Assumes the file is already locked in exclusive mode.
/// Atomically modifies the lock to be in shared mode, without releasing it.
///
/// TODO: integrate with async I/O
pub fn downgradeLock(file: File) LockError!void {
if (is_windows) {
// On Windows it works like a semaphore + exclusivity flag. To implement this
// function, we first obtain another lock in shared mode. This changes the
// exclusivity flag, but increments the semaphore to 2. So we follow up with
// an NtUnlockFile which decrements the semaphore but does not modify the
// exclusivity flag.
var io_status_block: windows.IO_STATUS_BLOCK = undefined;
windows.LockFile(
file.handle,
null,
null,
null,
&io_status_block,
&range_off,
&range_len,
null,
windows.TRUE, // non-blocking=true
windows.FALSE, // exclusive=false
) catch |err| switch (err) {
error.WouldBlock => unreachable, // File was not locked in exclusive mode.
else => |e| return e,
};
return windows.UnlockFile(
file.handle,
&io_status_block,
&range_off,
&range_len,
null,
) catch |err| switch (err) {
error.RangeNotLocked => unreachable, // File was not locked.
error.Unexpected => unreachable, // Resource deallocation must succeed.
};
} else {
return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
error.WouldBlock => unreachable, // File was not locked in exclusive mode.
else => |e| return e,
};
}
const non_blocking_flag = if (non_blocking) os.LOCK_NB else @as(i32, 0);
return os.flock(file.handle, switch (lock) {
.None => os.LOCK_UN,
.Shared => os.LOCK_SH | non_blocking_flag,
.Exclusive => os.LOCK_EX | non_blocking_flag,
});
}
};

89
lib/std/os/windows.zig
View File

@@ -49,7 +49,6 @@ pub const OpenFileOptions = struct {
dir: ?HANDLE = null,
sa: ?*SECURITY_ATTRIBUTES = null,
share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
share_access_nonblocking: bool = false,
creation: ULONG,
io_mode: std.io.ModeOverride,
/// If true, tries to open path as a directory.
@@ -60,8 +59,6 @@ pub const OpenFileOptions = struct {
follow_symlinks: bool = true,
};
/// TODO when share_access_nonblocking is false, this implementation uses
/// untinterruptible sleep() to block. This is not the final iteration of the API.
pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
return error.IsDir;
@@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
// If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
var delay: usize = 1;
while (true) {
const rc = ntdll.NtCreateFile(
&result,
options.access_mask,
&attr,
&io,
null,
FILE_ATTRIBUTE_NORMAL,
options.share_access,
options.creation,
flags,
null,
0,
);
switch (rc) {
.SUCCESS => {
if (std.io.is_async and options.io_mode == .evented) {
_ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
}
return result;
},
.OBJECT_NAME_INVALID => unreachable,
.OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
.OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
.NO_MEDIA_IN_DEVICE => return error.NoDevice,
.INVALID_PARAMETER => unreachable,
.SHARING_VIOLATION => {
if (options.share_access_nonblocking) {
return error.WouldBlock;
}
// TODO sleep in a way that is interruptable
// TODO integrate with async I/O
std.time.sleep(delay);
if (delay < 1 * std.time.ns_per_s) {
delay *= 2;
}
continue;
},
.ACCESS_DENIED => return error.AccessDenied,
.PIPE_BUSY => return error.PipeBusy,
.OBJECT_PATH_SYNTAX_BAD => unreachable,
.OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
.FILE_IS_A_DIRECTORY => return error.IsDir,
.NOT_A_DIRECTORY => return error.NotDir,
else => return unexpectedStatus(rc),
}
const rc = ntdll.NtCreateFile(
&result,
options.access_mask,
&attr,
&io,
null,
FILE_ATTRIBUTE_NORMAL,
options.share_access,
options.creation,
flags,
null,
0,
);
switch (rc) {
.SUCCESS => {
if (std.io.is_async and options.io_mode == .evented) {
_ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
}
return result;
},
.OBJECT_NAME_INVALID => unreachable,
.OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
.OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
.NO_MEDIA_IN_DEVICE => return error.NoDevice,
.INVALID_PARAMETER => unreachable,
.SHARING_VIOLATION => return error.AccessDenied,
.ACCESS_DENIED => return error.AccessDenied,
.PIPE_BUSY => return error.PipeBusy,
.OBJECT_PATH_SYNTAX_BAD => unreachable,
.OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
.FILE_IS_A_DIRECTORY => return error.IsDir,
.NOT_A_DIRECTORY => return error.NotDir,
else => return unexpectedStatus(rc),
}
}
@@ -1689,7 +1672,7 @@ pub fn LockFile(
Event: ?HANDLE,
ApcRoutine: ?*IO_APC_ROUTINE,
ApcContext: ?*c_void,
IoStatusBlock: ?*IO_STATUS_BLOCK,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
@@ -1712,6 +1695,7 @@ pub fn LockFile(
.SUCCESS => return,
.INSUFFICIENT_RESOURCES => return error.SystemResources,
.LOCK_NOT_GRANTED => return error.WouldBlock,
.ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
else => return unexpectedStatus(rc),
}
}
@@ -1722,7 +1706,7 @@ pub const UnlockFileError = error{
pub fn UnlockFile(
FileHandle: HANDLE,
IoStatusBlock: ?*IO_STATUS_BLOCK,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
@@ -1731,6 +1715,7 @@ pub fn UnlockFile(
switch (rc) {
.SUCCESS => return,
.RANGE_NOT_LOCKED => return error.RangeNotLocked,
.ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
else => return unexpectedStatus(rc),
}
}

4
lib/std/os/windows/ntdll.zig
View File

@@ -145,7 +145,7 @@ pub extern "NtDll" fn NtLockFile(
Event: ?HANDLE,
ApcRoutine: ?*IO_APC_ROUTINE,
ApcContext: ?*c_void,
IoStatusBlock: ?*IO_STATUS_BLOCK,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,
@@ -155,7 +155,7 @@ pub extern "NtDll" fn NtLockFile(
pub extern "NtDll" fn NtUnlockFile(
FileHandle: HANDLE,
IoStatusBlock: ?*IO_STATUS_BLOCK,
IoStatusBlock: *IO_STATUS_BLOCK,
ByteOffset: *const LARGE_INTEGER,
Length: *const LARGE_INTEGER,
Key: ?*ULONG,

8
src/Cache.zig
View File

@@ -670,15 +670,17 @@ pub const Manifest = struct {
fn downgradeToSharedLock(self: *Manifest) !void {
if (!self.have_exclusive_lock) return;
const manifest_file = self.manifest_file.?;
try manifest_file.setLock(.Shared, false);
try manifest_file.downgradeLock();
self.have_exclusive_lock = false;
}
fn upgradeToExclusiveLock(self: *Manifest) !void {
if (self.have_exclusive_lock) return;
const manifest_file = self.manifest_file.?;
try manifest_file.setLock(.None, false);
try manifest_file.setLock(.Exclusive, false);
// Here we intentionally have a period where the lock is released, in case there are
// other processes holding a shared lock.
manifest_file.unlock();
try manifest_file.lock(.Exclusive);
self.have_exclusive_lock = true;
}