notes

62.7 KB

1. // mpackdb DB engine — Zig port of mpackdb v1.0.7 (MPackDB.js / IndexManager.js)
2. // semantics. File-level compatible with the JS implementation (D12):
3. //
4. // <base>.mpack append-only records: [4-byte LE total size][msgpack map]
5. // <base>.meta.json { nextId?, deleted:[offsets], version, schema? }
6. // <base>.<field>.txt sorted index lines "key,offset,length\n"
7. // <base>.idxstate.json { coveredBytes }
8. // <base>.lock lock file containing the holder's pid (O_EXCL protocol,
9. // stale takeover after staleLockTimeout ms)
10. //
11. // Divergences from the JS implementation (documented in the 066 report):
12. // - the in-memory index is the COMPLETE entry set (JS keeps on-disk + delta);
13. // persist() writes the full set instead of read-merge-write. Sequential
14. // cross-process use (close one side, open the other) behaves identically.
15. // - string index keys sort bytewise (JS: String.localeCompare, ICU collation).
16. // Equality lookups are unaffected; range ordering can differ for
17. // mixed-case/accented keys.
18. // - null field values are not indexed (JS indexes them under the key "null").
19. // - external data-file replacement while open (inode change) is not detected.
20. // - a fresh `*id` table's first id is 1 (JS: 0) — ticket #4, see FIRST_ID.
22. const std = @import("std");
23. const linux = std.os.linux;
24. const msgpack = @import("msgpack.zig");
26. pub const PkType = enum(u8) { number = 0, uuid = 1, string = 2 };
28. /// A `*id` table's first auto-increment id (ticket #4, the creator's ruling:
29. /// "start at 1"). Only a table WITHOUT a stored counter starts here: a
30. /// .meta.json that carries `nextId` — every existing table, including one the
31. /// JS mpackdb wrote — keeps counting from it. The file format is unchanged;
32. /// the JS mpackdb 1.0.7 starts a fresh table at 0, which is the one divergence.
33. pub const FIRST_ID: f64 = 1;
34. pub const IdxType = enum(u8) { lexical = 0, numeric = 1 };
36. pub const Error = error{
37. NoDbFile,
38. NoPrimaryKey,
39. /// mission 077 (074 GAP 10): a query named a field with no index. There is
40. /// no index to binary-search and mpackdb does not silently full-scan, so
41. /// the answer used to be an empty result — indistinguishable from "no such
42. /// record". It is a mistake, and it says so now.
43. NoSuchIndex,
44. DuplicateKey,
45. /// ticket #29: update()'s record must carry the primary key. Without it
46. /// the old record was deleted, the new one written without a key, and the
47. /// table left corrupt (fetch null, a unique index still finding it).
48. RecordLacksPrimaryKey,
49. LockTimeout,
50. CorruptRecord,
51. IoError,
52. OutOfMemory,
53. InvalidFormat,
54. Truncated,
55. MapTooLarge,
56. };
58. pub const Key = union(enum) {
59. num: f64,
60. str: []const u8,
61. };
63. pub const Entry = struct {
64. key: Key,
65. off: u64,
66. len: u64,
67. };
69. const FieldIndex = struct {
70. typ: IdxType,
71. entries: std.ArrayList(Entry) = .empty, // sorted by (key, off)
72. };
74. pub const Loc = struct { off: u64, len: u64 };
76. pub const Record = struct {
77. value: msgpack.Value,
78. off: u64,
79. len: u64,
80. };
82. // =========================================================================
83. // Low-level file IO (raw linux syscalls — plugins avoid std.Io plumbing)
84. // =========================================================================
86. const fio = struct {
87. fn openz(alloc: std.mem.Allocator, path: []const u8, flags: linux.O, mode: u32) ?i32 {
88. const path_z = alloc.dupeZ(u8, path) catch return null;
89. defer alloc.free(path_z);
90. const rc = linux.open(path_z, flags, mode);
91. const fd: i32 = @bitCast(@as(u32, @truncate(rc)));
92. if (fd < 0) return null;
93. return fd;
94. }
96. fn openRead(alloc: std.mem.Allocator, path: []const u8) ?i32 {
97. return openz(alloc, path, .{}, 0);
98. }
100. fn openAppend(alloc: std.mem.Allocator, path: []const u8) ?i32 {
101. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .APPEND = true }, 0o644);
102. }
104. fn openTrunc(alloc: std.mem.Allocator, path: []const u8) ?i32 {
105. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .TRUNC = true }, 0o644);
106. }
108. /// O_CREAT|O_EXCL — returns null when the file already exists.
109. fn openExcl(alloc: std.mem.Allocator, path: []const u8) ?i32 {
110. return openz(alloc, path, .{ .ACCMODE = .WRONLY, .CREAT = true, .EXCL = true }, 0o644);
111. }
113. fn close(fd: i32) void {
114. _ = linux.close(fd);
115. }
117. fn writeAll(fd: i32, bytes: []const u8) bool {
118. var written: usize = 0;
119. while (written < bytes.len) {
120. const rc = linux.write(fd, bytes.ptr + written, bytes.len - written);
121. if (@as(isize, @bitCast(rc)) <= 0) return false;
122. written += rc;
123. }
124. return true;
125. }
127. fn pread(fd: i32, buf: []u8, offset: u64) ?usize {
128. var got: usize = 0;
129. while (got < buf.len) {
130. const rc = linux.pread(fd, buf.ptr + got, buf.len - got, @intCast(offset + got));
131. const n: isize = @bitCast(rc);
132. if (n < 0) return null;
133. if (n == 0) break;
134. got += @intCast(n);
135. }
136. return got;
137. }
139. fn readAll(alloc: std.mem.Allocator, path: []const u8) ?[]u8 {
140. const fd = openRead(alloc, path) orelse return null;
141. defer close(fd);
142. var content: std.ArrayList(u8) = .empty;
143. var buf: [65536]u8 = undefined;
144. while (true) {
145. const rc = linux.read(fd, &buf, buf.len);
146. if (@as(isize, @bitCast(rc)) <= 0) break;
147. content.appendSlice(alloc, buf[0..rc]) catch return null;
148. }
149. return content.toOwnedSlice(alloc) catch null;
150. }
152. fn fileSize(alloc: std.mem.Allocator, path: []const u8) ?u64 {
153. const path_z = alloc.dupeZ(u8, path) catch return null;
154. defer alloc.free(path_z);
155. var stx: linux.Statx = undefined;
156. const rc = linux.statx(linux.AT.FDCWD, path_z, 0, linux.STATX.BASIC_STATS, &stx);
157. if (rc != 0) return null;
158. return stx.size;
159. }
161. fn mtimeMs(alloc: std.mem.Allocator, path: []const u8) ?i64 {
162. const path_z = alloc.dupeZ(u8, path) catch return null;
163. defer alloc.free(path_z);
164. var stx: linux.Statx = undefined;
165. const rc = linux.statx(linux.AT.FDCWD, path_z, 0, linux.STATX.BASIC_STATS, &stx);
166. if (rc != 0) return null;
167. return @as(i64, stx.mtime.sec) * 1000 + @divTrunc(@as(i64, stx.mtime.nsec), 1_000_000);
168. }
170. fn rename(alloc: std.mem.Allocator, old_path: []const u8, new_path: []const u8) bool {
171. const old_z = alloc.dupeZ(u8, old_path) catch return false;
172. defer alloc.free(old_z);
173. const new_z = alloc.dupeZ(u8, new_path) catch return false;
174. defer alloc.free(new_z);
175. return linux.renameat(linux.AT.FDCWD, old_z, linux.AT.FDCWD, new_z) == 0;
176. }
178. fn unlink(alloc: std.mem.Allocator, path: []const u8) bool {
179. const path_z = alloc.dupeZ(u8, path) catch return false;
180. defer alloc.free(path_z);
181. return linux.unlinkat(linux.AT.FDCWD, path_z, 0) == 0;
182. }
184. fn mkdirAll(alloc: std.mem.Allocator, dir_path: []const u8) void {
185. if (dir_path.len == 0 or std.mem.eql(u8, dir_path, ".")) return;
186. var i: usize = 1;
187. while (i <= dir_path.len) : (i += 1) {
188. if (i == dir_path.len or dir_path[i] == '/') {
189. const part = alloc.dupeZ(u8, dir_path[0..i]) catch return;
190. defer alloc.free(part);
191. _ = linux.mkdirat(linux.AT.FDCWD, part, 0o755);
192. }
193. }
194. }
196. fn sleepMs(ms: u64) void {
197. var req: linux.timespec = .{ .sec = @intCast(ms / 1000), .nsec = @intCast((ms % 1000) * 1_000_000) };
198. var rem: linux.timespec = undefined;
199. _ = linux.nanosleep(&req, &rem);
200. }
202. fn nowMs() i64 {
203. var ts: linux.timespec = undefined;
204. _ = linux.clock_gettime(linux.CLOCK.REALTIME, &ts);
205. return @as(i64, ts.sec) * 1000 + @divTrunc(@as(i64, ts.nsec), 1_000_000);
206. }
207. };
209. // =========================================================================
210. // JS-compatible helpers
211. // =========================================================================
213. /// Format an f64 the way JS String(n) does for the values mpackdb produces:
214. /// integral values print without a decimal point. (Exotic floats may diverge
215. /// from V8's shortest-round-trip formatting; index keys are ints/strings in
216. /// practice.)
217. pub fn jsNumFmt(buf: []u8, v: f64) []const u8 {
218. if (v == @floor(v) and @abs(v) <= 9007199254740992.0) {
219. const i: i64 = @intFromFloat(v);
220. return std.fmt.bufPrint(buf, "{d}", .{i}) catch buf[0..0];
221. }
222. return std.fmt.bufPrint(buf, "{d}", .{v}) catch buf[0..0];
223. }
225. /// JS parseInt(s, 10) semantics: optional sign, leading digits, ignore rest.
226. /// Returns null for NaN.
227. fn jsParseInt(s: []const u8) ?f64 {
228. var i: usize = 0;
229. while (i < s.len and (s[i] == ' ' or s[i] == '\t')) i += 1;
230. var sign: f64 = 1;
231. if (i < s.len and (s[i] == '+' or s[i] == '-')) {
232. if (s[i] == '-') sign = -1;
233. i += 1;
234. }
235. var got = false;
236. var v: f64 = 0;
237. while (i < s.len and s[i] >= '0' and s[i] <= '9') : (i += 1) {
238. v = v * 10 + @as(f64, @floatFromInt(s[i] - '0'));
239. got = true;
240. }
241. if (!got) return null;
242. return sign * v;
243. }
245. /// _compareKeys: numbers numerically; otherwise stringified byte compare
246. /// (JS uses localeCompare — see divergence note in the header).
247. pub fn cmpKeys(a: Key, b: Key) i32 {
248. if (a == .num and b == .num) {
249. if (a.num < b.num) return -1;
250. if (a.num > b.num) return 1;
251. return 0;
252. }
253. var buf_a: [32]u8 = undefined;
254. var buf_b: [32]u8 = undefined;
255. const sa = if (a == .str) a.str else jsNumFmt(&buf_a, a.num);
256. const sb = if (b == .str) b.str else jsNumFmt(&buf_b, b.num);
257. return switch (std.mem.order(u8, sa, sb)) {
258. .lt => -1,
259. .eq => 0,
260. .gt => 1,
261. };
262. }
264. fn entryLess(_: void, a: Entry, b: Entry) bool {
265. const c = cmpKeys(a.key, b.key);
266. if (c != 0) return c < 0;
267. return a.off < b.off;
268. }
270. /// mpack.js uuid(): 9-char base36 ms timestamp (padStart '0') + 3 base36 chars.
271. pub fn genUuid(buf: *[12]u8) []const u8 {
272. const digits = "0123456789abcdefghijklmnopqrstuvwxyz";
273. const t: u64 = @intCast(fio.nowMs());
274. var tmp: [16]u8 = undefined;
275. var n: usize = 0;
276. var v = t;
277. while (v > 0) : (v /= 36) {
278. tmp[n] = digits[@intCast(v % 36)];
279. n += 1;
280. }
281. var i: usize = 0;
282. while (i < 9) : (i += 1) {
283. buf[8 - i] = if (i < n) tmp[i] else '0';
284. }
285. if (!rng_init) {
286. var ts: linux.timespec = undefined;
287. _ = linux.clock_gettime(linux.CLOCK.MONOTONIC, &ts);
288. rng = std.Random.DefaultPrng.init(@bitCast(@as(i64, ts.sec) *% 1_000_000_000 +% ts.nsec));
289. rng_init = true;
290. }
291. var r = rng.random();
292. for (0..3) |j| {
293. buf[9 + j] = digits[r.uintLessThan(u8, 36)];
294. }
295. return buf[0..12];
296. }
298. var rng: std.Random.DefaultPrng = .{ .s = undefined };
299. var rng_init: bool = false;
301. // =========================================================================
302. // The database
303. // =========================================================================
305. pub const OpenOptions = struct {
306. primary_key: ?[]const u8 = null, // with */@/! prefixes, like the JS API
307. indexes: []const []const u8 = &.{}, // with prefixes
308. compact: bool = true,
309. stale_lock_timeout_ms: i64 = 30000,
310. debug: bool = false,
311. };
313. /// The identity of a table on disk (ticket #110): dirname + basename with any
314. /// extension stripped, exactly the computation `Db.open` uses to turn a given
315. /// path into `<base>.mpack` — so two spellings of the same file (`x.db`, `x`)
316. /// share one key. The plugin ABI layer (mpackdb.zig) keys its process-wide
317. /// handle registry on this, so a table opened twice in one process — from any
318. /// module instance or realm — shares the one live `Db` instead of each open
319. /// racing the other's file state.
320. pub fn tableKey(alloc: std.mem.Allocator, db_file: []const u8) Error![]u8 {
321. const dir = std.fs.path.dirname(db_file) orelse ".";
322. var base = std.fs.path.basename(db_file);
323. if (std.mem.lastIndexOfScalar(u8, base, '.')) |dot| {
324. if (dot > 0) base = base[0..dot];
325. }
326. return std.fmt.allocPrint(alloc, "{s}/{s}", .{ dir, base }) catch Error.OutOfMemory;
327. }
329. pub const Db = struct {
330. arena_state: std.heap.ArenaAllocator,
331. alloc: std.mem.Allocator, // arena — freed wholesale on close
333. // paths
334. data_path: []const u8,
335. meta_path: []const u8,
336. lock_path: []const u8,
337. idxstate_path: []const u8,
339. // schema
340. pk: ?[]const u8 = null,
341. pk_type: PkType = .string,
342. index_fields: std.ArrayList([]const u8) = .empty, // pk first (when set), like JS
343. unique_fields: std.ArrayList([]const u8) = .empty,
344. idx: std.StringArrayHashMapUnmanaged(FieldIndex) = .empty,
346. // meta
347. next_id: f64 = FIRST_ID,
348. deleted: std.ArrayList(u64) = .empty,
349. version: u64 = 0,
351. covered_bytes: u64 = 0,
352. compact_on_open: bool = true,
353. stale_lock_timeout_ms: i64 = 30000,
354. debug: bool = false,
356. // last DuplicateKey detail for the plugin surface
357. last_error_buf: [256]u8 = undefined,
358. last_error: []const u8 = "",
360. pub fn open(gpa: std.mem.Allocator, db_file: []const u8, opts: OpenOptions) Error!*Db {
361. if (db_file.len == 0) return Error.NoDbFile;
363. const self = gpa.create(Db) catch return Error.OutOfMemory;
364. self.* = .{
365. .arena_state = std.heap.ArenaAllocator.init(gpa),
366. .alloc = undefined,
367. .data_path = undefined,
368. .meta_path = undefined,
369. .lock_path = undefined,
370. .idxstate_path = undefined,
371. };
372. self.alloc = self.arena_state.allocator();
373. errdefer {
374. self.arena_state.deinit();
375. gpa.destroy(self);
376. }
377. const a = self.alloc;
379. self.compact_on_open = opts.compact;
380. self.stale_lock_timeout_ms = opts.stale_lock_timeout_ms;
381. self.debug = opts.debug;
383. // dirname / basename (extension stripped, like JS basename(f, extname(f)))
384. const dir = std.fs.path.dirname(db_file) orelse ".";
385. var base = std.fs.path.basename(db_file);
386. if (std.mem.lastIndexOfScalar(u8, base, '.')) |dot| {
387. if (dot > 0) base = base[0..dot];
388. }
389. fio.mkdirAll(a, dir);
391. self.data_path = std.fmt.allocPrint(a, "{s}/{s}.mpack", .{ dir, base }) catch return Error.OutOfMemory;
392. self.meta_path = std.fmt.allocPrint(a, "{s}/{s}.meta.json", .{ dir, base }) catch return Error.OutOfMemory;
393. self.lock_path = std.fmt.allocPrint(a, "{s}/{s}.lock", .{ dir, base }) catch return Error.OutOfMemory;
394. self.idxstate_path = std.fmt.allocPrint(a, "{s}/{s}.idxstate.json", .{ dir, base }) catch return Error.OutOfMemory;
396. // ---- parse primary key ----
397. if (opts.primary_key) |pk_raw| {
398. if (pk_raw.len > 0) {
399. if (pk_raw[0] == '*') {
400. self.pk = a.dupe(u8, pk_raw[1..]) catch return Error.OutOfMemory;
401. self.pk_type = .number;
402. } else if (pk_raw[0] == '@') {
403. self.pk = a.dupe(u8, pk_raw[1..]) catch return Error.OutOfMemory;
404. self.pk_type = .uuid;
405. } else {
406. self.pk = a.dupe(u8, pk_raw) catch return Error.OutOfMemory;
407. self.pk_type = .string;
408. }
409. }
410. }
411. if (self.pk) |p| self.unique_fields.append(a, p) catch return Error.OutOfMemory;
413. // ---- parse indexes (pk first, then configured; dedupe) ----
414. if (self.pk) |p| {
415. self.index_fields.append(a, p) catch return Error.OutOfMemory;
416. const t: IdxType = if (self.pk_type == .number) .numeric else .lexical;
417. self.idx.put(a, p, .{ .typ = t }) catch return Error.OutOfMemory;
418. }
419. for (opts.indexes) |raw| {
420. var clean = raw;
421. var is_unique = false;
422. if (clean.len > 0 and clean[0] == '!') {
423. is_unique = true;
424. clean = clean[1..];
425. }
426. var typ: IdxType = .lexical;
427. if (clean.len > 0 and clean[0] == '*') {
428. typ = .numeric;
429. clean = clean[1..];
430. } else if (clean.len > 0 and clean[0] == '@') {
431. clean = clean[1..];
432. }
433. if (self.idx.contains(clean)) continue;
434. const owned = a.dupe(u8, clean) catch return Error.OutOfMemory;
435. self.index_fields.append(a, owned) catch return Error.OutOfMemory;
436. self.idx.put(a, owned, .{ .typ = typ }) catch return Error.OutOfMemory;
437. if (is_unique) self.unique_fields.append(a, owned) catch return Error.OutOfMemory;
438. }
440. // ---- load meta ----
441. self.loadMeta();
443. // ---- compact on init (JS default; also creates an empty data file) ----
444. // AN OPEN WITH NOTHING TO CHANGE WRITES NOTHING (ticket #21): a store
445. // with no tombstones has nothing to compact, and rewriting it anyway
446. // made opening a backup change it.
447. var did_compact = false;
448. const nothing_to_compact = self.deleted.items.len == 0 and
449. fio.fileSize(self.alloc, self.data_path) != null;
450. if (self.compact_on_open and !nothing_to_compact) {
451. try self.acquireLock();
452. const cr = self.compactLocked();
453. self.releaseLock();
454. try cr;
455. did_compact = true;
456. }
458. // ---- persist schema into meta (JS: init-schema under lock).
459. // compactLocked already wrote meta (with schema); skip the extra bump.
460. if (!did_compact and (self.pk != null or self.index_fields.items.len > 0) and
461. !self.metaOnDiskIsCurrent())
462. {
463. try self.acquireLock();
464. const mr = self.persistMetaLocked();
465. self.releaseLock();
466. try mr;
467. }
469. // ---- indexes (JS: (re)build under the lock; force after compaction) ----
470. if (self.index_fields.items.len > 0) {
471. try self.acquireLock();
472. const ir = self.initIndexes(did_compact);
473. self.releaseLock();
474. try ir;
475. }
477. return self;
478. }
480. pub fn close(self: *Db) void {
481. // persist indexes + idxstate (JS: IndexManager.close → persist under lock)
482. if (self.index_fields.items.len > 0) {
483. if (self.acquireLock()) {
484. self.persistIndexesLocked() catch {};
485. self.releaseLock();
486. } else |_| {}
487. }
488. const gpa = self.arena_state.child_allocator;
489. self.arena_state.deinit();
490. gpa.destroy(self);
491. }
493. fn dbg(self: *Db, comptime fmt: []const u8, args: anytype) void {
494. if (self.debug) std.debug.print("[mpackdb] " ++ fmt ++ "\n", args);
495. }
497. // =====================================================================
498. // Locking (JS _acquireFileLock protocol)
499. // =====================================================================
501. fn acquireLock(self: *Db) Error!void {
502. var retries: u32 = 0;
503. while (true) {
504. if (fio.openExcl(self.alloc, self.lock_path)) |fd| {
505. var pid_buf: [16]u8 = undefined;
506. const pid_s = std.fmt.bufPrint(&pid_buf, "{d}", .{linux.getpid()}) catch "0";
507. _ = fio.writeAll(fd, pid_s);
508. fio.close(fd);
509. return;
510. }
511. // stale lock takeover
512. if (self.stale_lock_timeout_ms > 0) {
513. if (fio.mtimeMs(self.alloc, self.lock_path)) |mt| {
514. if (fio.nowMs() - mt > self.stale_lock_timeout_ms) {
515. _ = fio.unlink(self.alloc, self.lock_path);
516. continue;
517. }
518. }
519. }
520. if (retries > 480) return Error.LockTimeout; // ~12s at 25ms
521. fio.sleepMs(25);
522. retries += 1;
523. }
524. }
526. fn releaseLock(self: *Db) void {
527. _ = fio.unlink(self.alloc, self.lock_path);
528. }
530. // =====================================================================
531. // Meta (.meta.json)
532. // =====================================================================
534. fn appendFmt(self: *Db, list: *std.ArrayList(u8), comptime fmt: []const u8, args: anytype) Error!void {
535. const s = std.fmt.allocPrint(self.alloc, fmt, args) catch return Error.OutOfMemory;
536. defer self.alloc.free(s);
537. list.appendSlice(self.alloc, s) catch return Error.OutOfMemory;
538. }
540. fn loadMeta(self: *Db) void {
541. const content = fio.readAll(self.alloc, self.meta_path) orelse return;
542. self.applyMetaJson(content);
543. }
545. fn applyMetaJson(self: *Db, content: []const u8) void {
546. const parsed = std.json.parseFromSlice(std.json.Value, self.alloc, content, .{}) catch return;
547. defer parsed.deinit();
548. if (parsed.value != .object) return;
549. const obj = parsed.value.object;
550. self.next_id = FIRST_ID;
551. self.deleted.clearRetainingCapacity();
552. self.version = 0;
553. if (obj.get("nextId")) |v| {
554. self.next_id = switch (v) {
555. .integer => |i| @floatFromInt(i),
556. .float => |f| f,
557. else => FIRST_ID,
558. };
559. }
560. if (obj.get("version")) |v| {
561. if (v == .integer) self.version = @intCast(@max(v.integer, 0));
562. }
563. if (obj.get("deleted")) |v| {
564. if (v == .array) {
565. for (v.array.items) |it| {
566. const off: u64 = switch (it) {
567. .integer => |i| @intCast(@max(i, 0)),
568. .float => |f| @intFromFloat(@max(f, 0)),
569. else => continue,
570. };
571. self.deleted.append(self.alloc, off) catch {};
572. }
573. }
574. }
575. }
577. /// Pick up other processes' persisted state (JS refresh()): adopt disk meta
578. /// when its version is newer; index appended tail records.
579. pub fn refresh(self: *Db) void {
580. if (fio.readAll(self.alloc, self.meta_path)) |content| {
581. defer self.alloc.free(content);
582. const parsed = std.json.parseFromSlice(std.json.Value, self.alloc, content, .{}) catch return;
583. defer parsed.deinit();
584. if (parsed.value == .object) {
585. var disk_version: u64 = 0;
586. if (parsed.value.object.get("version")) |v| {
587. if (v == .integer) disk_version = @intCast(@max(v.integer, 0));
588. }
589. if (disk_version > self.version) self.applyMetaJson(content);
590. }
591. }
592. // catchUp: index records another process appended
593. if (self.index_fields.items.len > 0) {
594. const size = fio.fileSize(self.alloc, self.data_path) orelse 0;
595. if (size > self.covered_bytes) self.catchUp(size);
596. }
597. }
599. fn persistMetaLocked(self: *Db) Error!void {
600. self.version += 1;
601. var out: std.ArrayList(u8) = .empty;
602. defer out.deinit(self.alloc);
603. try self.renderMeta(&out);
605. // atomic tmp + rename (JS: `${metaPath}.${pid}.tmp`)
606. var tmp_buf: [512]u8 = undefined;
607. const tmp_path = std.fmt.bufPrint(&tmp_buf, "{s}.{d}.tmp", .{ self.meta_path, linux.getpid() }) catch return Error.IoError;
608. const fd = fio.openTrunc(self.alloc, tmp_path) orelse return Error.IoError;
609. const ok = fio.writeAll(fd, out.items);
610. fio.close(fd);
611. if (!ok) return Error.IoError;
612. if (!fio.rename(self.alloc, tmp_path, self.meta_path)) return Error.IoError;
613. }
615. /// Does .meta.json already say, byte for byte, what this handle would write
616. /// at its current version? Then an open has nothing to persist (ticket #21).
617. fn metaOnDiskIsCurrent(self: *Db) bool {
618. const disk = fio.readAll(self.alloc, self.meta_path) orelse return false;
619. defer self.alloc.free(disk);
620. var out: std.ArrayList(u8) = .empty;
621. defer out.deinit(self.alloc);
622. self.renderMeta(&out) catch return false;
623. return std.mem.eql(u8, disk, out.items);
624. }
626. fn renderMeta(self: *Db, out: *std.ArrayList(u8)) Error!void {
627. try self.appendFmt(out, "{{", .{});
628. if (self.pk != null and self.pk_type == .number) {
629. var nbuf: [32]u8 = undefined;
630. try self.appendFmt(out, "\"nextId\":{s},", .{jsNumFmt(&nbuf, self.next_id)});
631. }
632. try self.appendFmt(out, "\"deleted\":[", .{});
633. for (self.deleted.items, 0..) |off, i| {
634. if (i > 0) try self.appendFmt(out, ",", .{});
635. try self.appendFmt(out, "{d}", .{off});
636. }
637. try self.appendFmt(out, "],\"version\":{d}", .{self.version});
638. if (self.pk != null or self.index_fields.items.len > 0) {
639. try self.appendFmt(out, ",\"schema\":{{", .{});
640. var first = true;
641. if (self.pk) |p| {
642. const prefix: []const u8 = switch (self.pk_type) {
643. .number => "*",
644. .uuid => "@",
645. .string => "",
646. };
647. try self.appendFmt(out, "\"primaryKey\":\"{s}{s}\"", .{ prefix, p });
648. first = false;
649. }
650. if (!first) try self.appendFmt(out, ",", .{});
651. try self.appendFmt(out, "\"indexes\":[", .{});
652. var n: usize = 0;
653. for (self.index_fields.items) |field| {
654. if (self.pk != null and std.mem.eql(u8, field, self.pk.?)) continue;
655. if (n > 0) try self.appendFmt(out, ",", .{});
656. const uni = for (self.unique_fields.items) |u| {
657. if (std.mem.eql(u8, u, field)) break true;
658. } else false;
659. const numeric = (self.idx.get(field) orelse FieldIndex{ .typ = .lexical }).typ == .numeric;
660. try self.appendFmt(out, "\"{s}{s}{s}\"", .{
661. if (uni) "!" else "",
662. if (numeric) "*" else "",
663. field,
664. });
665. n += 1;
666. }
667. try self.appendFmt(out, "]}}", .{});
668. }
669. try self.appendFmt(out, "}}", .{});
670. }
672. // =====================================================================
673. // Data file scanning
674. // =====================================================================
676. fn deletedSet(self: *Db, alloc: std.mem.Allocator) std.AutoHashMapUnmanaged(u64, void) {
677. var set: std.AutoHashMapUnmanaged(u64, void) = .empty;
678. for (self.deleted.items) |off| set.put(alloc, off, {}) catch {};
679. return set;
680. }
682. /// Sequential scan yielding all live records. Caller supplies an arena for
683. /// decoded values. Used for rebuilds, compaction and unindexed finds.
684. pub const Scanner = struct {
685. db: *Db,
686. fd: i32 = -1,
687. offset: u64 = 0,
688. size: u64 = 0,
689. skip_deleted: bool = true,
690. deleted_set: std.AutoHashMapUnmanaged(u64, void) = .empty,
691. scratch: std.mem.Allocator,
693. pub fn init(db: *Db, scratch: std.mem.Allocator, skip_deleted: bool) Scanner {
694. var s = Scanner{ .db = db, .scratch = scratch, .skip_deleted = skip_deleted };
695. s.size = fio.fileSize(scratch, db.data_path) orelse 0;
696. if (s.size > 0) {
697. s.fd = fio.openRead(scratch, db.data_path) orelse -1;
698. }
699. if (skip_deleted) s.deleted_set = db.deletedSet(scratch);
700. return s;
701. }
703. pub fn deinit(self: *Scanner) void {
704. if (self.fd >= 0) fio.close(self.fd);
705. self.fd = -1;
706. }
708. /// Returns the next record (decoded into `arena`) or null at EOF.
709. pub fn next(self: *Scanner, arena: std.mem.Allocator) Error!?Record {
710. while (true) {
711. if (self.fd < 0 or self.offset + 4 > self.size) return null;
712. var hdr: [4]u8 = undefined;
713. const got = fio.pread(self.fd, &hdr, self.offset) orelse return Error.IoError;
714. if (got < 4) return null;
715. const rec_size = std.mem.readInt(u32, &hdr, .little);
716. if (rec_size <= 4) return Error.CorruptRecord;
717. if (self.offset + rec_size > self.size) return null; // incomplete tail
718. const off = self.offset;
719. self.offset += rec_size;
720. if (self.skip_deleted and self.deleted_set.contains(off)) continue;
721. const buf = arena.alloc(u8, rec_size - 4) catch return Error.OutOfMemory;
722. const got2 = fio.pread(self.fd, buf, off + 4) orelse return Error.IoError;
723. if (got2 < rec_size - 4) return Error.Truncated;
724. const d = msgpack.decode(arena, buf) catch return Error.CorruptRecord;
725. return Record{ .value = d.value, .off = off, .len = rec_size };
726. }
727. }
729. /// Like next() but without decoding — yields the raw framed bytes.
730. pub fn nextRaw(self: *Scanner, arena: std.mem.Allocator) Error!?struct { bytes: []u8, off: u64 } {
731. while (true) {
732. if (self.fd < 0 or self.offset + 4 > self.size) return null;
733. var hdr: [4]u8 = undefined;
734. const got = fio.pread(self.fd, &hdr, self.offset) orelse return Error.IoError;
735. if (got < 4) return null;
736. const rec_size = std.mem.readInt(u32, &hdr, .little);
737. if (rec_size <= 4) return Error.CorruptRecord;
738. if (self.offset + rec_size > self.size) return null;
739. const off = self.offset;
740. self.offset += rec_size;
741. if (self.skip_deleted and self.deleted_set.contains(off)) continue;
742. const buf = arena.alloc(u8, rec_size) catch return Error.OutOfMemory;
743. const got2 = fio.pread(self.fd, buf, off) orelse return Error.IoError;
744. if (got2 < rec_size) return Error.Truncated;
745. return .{ .bytes = buf, .off = off };
746. }
747. }
748. };
750. /// Read + decode one record by location.
751. pub fn readAt(self: *Db, arena: std.mem.Allocator, loc: Loc) Error!msgpack.Value {
752. const fd = fio.openRead(self.alloc, self.data_path) orelse return Error.IoError;
753. defer fio.close(fd);
754. if (loc.len <= 4) return Error.CorruptRecord;
755. const buf = arena.alloc(u8, loc.len - 4) catch return Error.OutOfMemory;
756. const got = fio.pread(fd, buf, loc.off + 4) orelse return Error.IoError;
757. if (got < buf.len) return Error.Truncated;
758. const d = msgpack.decode(arena, buf) catch return Error.CorruptRecord;
759. return d.value;
760. }
762. // =====================================================================
763. // Index management
764. // =====================================================================
766. fn keyFromValue(v: msgpack.Value) ?Key {
767. return switch (v) {
768. .number => |n| Key{ .num = n },
769. .str => |s| Key{ .str = s },
770. else => null, // null/bool/objects are not indexed (see header note)
771. };
772. }
774. /// Duplicate a key's string into the db arena so it outlives the op arena.
775. fn ownKey(self: *Db, k: Key) Error!Key {
776. return switch (k) {
777. .num => k,
778. .str => |s| Key{ .str = self.alloc.dupe(u8, s) catch return Error.OutOfMemory },
779. };
780. }
782. fn lowerBound(entries: []const Entry, key: Key) usize {
783. var lo: usize = 0;
784. var hi: usize = entries.len;
785. while (lo < hi) {
786. const mid = lo + (hi - lo) / 2;
787. if (cmpKeys(entries[mid].key, key) < 0) {
788. lo = mid + 1;
789. } else {
790. hi = mid;
791. }
792. }
793. return lo;
794. }
796. /// Binary search: all entries whose key equals `key` (appended to `out`).
797. pub fn indexGet(self: *Db, field: []const u8, key: Key, alloc: std.mem.Allocator, out: *std.ArrayList(Entry)) Error!void {
798. const fi = self.idx.getPtr(field) orelse return Error.NoSuchIndex;
799. const parsed = self.parseKeyForField(field, key);
800. const entries = fi.entries.items;
801. var i = lowerBound(entries, parsed);
802. while (i < entries.len and cmpKeys(entries[i].key, parsed) == 0) : (i += 1) {
803. out.append(alloc, entries[i]) catch return Error.OutOfMemory;
804. }
805. }
807. /// Range scan [from, to] (either side optional), ascending.
808. pub fn indexRange(self: *Db, field: []const u8, from: ?Key, to: ?Key, alloc: std.mem.Allocator, out: *std.ArrayList(Entry)) Error!void {
809. const fi = self.idx.getPtr(field) orelse return Error.NoSuchIndex;
810. const entries = fi.entries.items;
811. var i: usize = if (from) |f| lowerBound(entries, self.parseKeyForField(field, f)) else 0;
812. const to_key: ?Key = if (to) |t| self.parseKeyForField(field, t) else null;
813. while (i < entries.len) : (i += 1) {
814. if (to_key) |t| {
815. if (cmpKeys(entries[i].key, t) > 0) break;
816. }
817. out.append(alloc, entries[i]) catch return Error.OutOfMemory;
818. }
819. }
821. /// _parseKey semantics: numeric index fields parse string keys via
822. /// parseInt; on NaN the key stays a string.
823. fn parseKeyForField(self: *Db, field: []const u8, key: Key) Key {
824. const fi = self.idx.get(field) orelse return key;
825. if (fi.typ == .numeric and key == .str) {
826. if (jsParseInt(key.str)) |n| return Key{ .num = n };
827. }
828. return key;
829. }
831. fn indexInsertEntry(self: *Db, field: []const u8, key: Key, loc: Loc) Error!void {
832. const fi = self.idx.getPtr(field) orelse return;
833. const owned = try self.ownKey(key);
834. const e = Entry{ .key = owned, .off = loc.off, .len = loc.len };
835. // insert at sorted position
836. const pos = blk: {
837. var lo: usize = 0;
838. var hi: usize = fi.entries.items.len;
839. while (lo < hi) {
840. const mid = lo + (hi - lo) / 2;
841. if (entryLess({}, fi.entries.items[mid], e)) {
842. lo = mid + 1;
843. } else {
844. hi = mid;
845. }
846. }
847. break :blk lo;
848. };
849. fi.entries.insert(self.alloc, pos, e) catch return Error.OutOfMemory;
850. }
852. /// Index a record's fields at loc.
853. fn indexInsertRecord(self: *Db, rec: msgpack.Value, loc: Loc) Error!void {
854. for (self.index_fields.items) |field| {
855. const v = rec.get(field) orelse continue;
856. const key = keyFromValue(v) orelse continue;
857. try self.indexInsertEntry(field, key, loc);
858. }
859. self.covered_bytes = @max(self.covered_bytes, loc.off + loc.len);
860. }
862. /// Remove all entries pointing at offset `off` (all fields).
863. fn indexRemoveOffset(self: *Db, off: u64) void {
864. var it = self.idx.iterator();
865. while (it.next()) |kv| {
866. const list = &kv.value_ptr.entries;
867. var i: usize = 0;
868. while (i < list.items.len) {
869. if (list.items[i].off == off) {
870. _ = list.orderedRemove(i);
871. } else {
872. i += 1;
873. }
874. }
875. }
876. }
878. fn indexPath(self: *Db, buf: []u8, field: []const u8) []const u8 {
879. // <dir>/<base>.<field>.txt — derive from idxstate path (…/base.idxstate.json)
880. const prefix = self.idxstate_path[0 .. self.idxstate_path.len - "idxstate.json".len];
881. return std.fmt.bufPrint(buf, "{s}{s}.txt", .{ prefix, field }) catch buf[0..0];
882. }
884. fn initIndexes(self: *Db, force_rebuild: bool) Error!void {
885. var need_rebuild = force_rebuild;
886. const data_size = fio.fileSize(self.alloc, self.data_path) orelse 0;
888. if (!need_rebuild) {
889. for (self.index_fields.items) |field| {
890. var pbuf: [512]u8 = undefined;
891. const p = self.indexPath(&pbuf, field);
892. const isize_ = fio.fileSize(self.alloc, p);
893. if (isize_ == null or isize_.? == 0) {
894. if (data_size > 0) {
895. need_rebuild = true;
896. break;
897. }
898. }
899. }
900. }
901. if (!need_rebuild) {
902. // coveredBytes from idxstate — no state file means rebuild (JS)
903. if (self.readIdxState()) |cb| {
904. self.covered_bytes = cb;
905. } else {
906. need_rebuild = true;
907. }
908. }
910. if (need_rebuild) {
911. try self.rebuildIndexes();
912. return;
913. }
915. // Load index files into memory
916. for (self.index_fields.items) |field| {
917. var pbuf: [512]u8 = undefined;
918. const p = self.indexPath(&pbuf, field);
919. const content = fio.readAll(self.alloc, p) orelse continue;
920. defer self.alloc.free(content);
921. self.loadIndexLines(field, content);
922. }
923. // sort (files are sorted by JS localeCompare; re-sort under our order)
924. var it = self.idx.iterator();
925. while (it.next()) |kv| {
926. std.sort.pdq(Entry, kv.value_ptr.entries.items, {}, entryLess);
927. }
928. if (data_size > self.covered_bytes) self.catchUp(data_size);
929. }
931. fn loadIndexLines(self: *Db, field: []const u8, content: []const u8) void {
932. const fi = self.idx.getPtr(field) orelse return;
933. var lines = std.mem.splitScalar(u8, content, '\n');
934. while (lines.next()) |line| {
935. if (line.len == 0) continue;
936. // key = up to FIRST comma (JS split(',')[0]); then offset, length
937. const c1 = std.mem.indexOfScalar(u8, line, ',') orelse continue;
938. const rest = line[c1 + 1 ..];
939. const c2 = std.mem.indexOfScalar(u8, rest, ',') orelse continue;
940. const off_s = rest[0..c2];
941. const len_s = rest[c2 + 1 ..];
942. const off = std.fmt.parseInt(u64, off_s, 10) catch continue;
943. const len = std.fmt.parseInt(u64, len_s, 10) catch continue;
944. if (len == 0) continue;
945. const key_s = line[0..c1];
946. var key: Key = undefined;
947. if (fi.typ == .numeric) {
948. key = if (jsParseInt(key_s)) |n| Key{ .num = n } else Key{ .str = self.alloc.dupe(u8, key_s) catch return };
949. } else {
950. key = Key{ .str = self.alloc.dupe(u8, key_s) catch return };
951. }
952. fi.entries.append(self.alloc, .{ .key = key, .off = off, .len = len }) catch return;
953. }
954. }
956. fn rebuildIndexes(self: *Db) Error!void {
957. var it0 = self.idx.iterator();
958. while (it0.next()) |kv| kv.value_ptr.entries.clearRetainingCapacity();
960. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
961. defer scratch_state.deinit();
962. const scratch = scratch_state.allocator();
964. // JS rebuild scans ALL records (no tombstone filter — deleted offsets
965. // simply get filtered at read time). Mirror that.
966. var scanner = Scanner.init(self, scratch, false);
967. defer scanner.deinit();
968. while (try scanner.next(scratch)) |rec| {
969. try self.indexInsertRecord(rec.value, .{ .off = rec.off, .len = rec.len });
970. }
971. self.covered_bytes = fio.fileSize(self.alloc, self.data_path) orelse 0;
972. try self.writeIndexFiles();
973. try self.writeIdxState();
974. }
976. fn catchUp(self: *Db, file_size: u64) void {
977. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
978. defer scratch_state.deinit();
979. const scratch = scratch_state.allocator();
981. var scanner = Scanner.init(self, scratch, false);
982. defer scanner.deinit();
983. scanner.offset = self.covered_bytes;
984. while (true) {
985. const maybe = scanner.next(scratch) catch break;
986. const rec = maybe orelse break;
987. self.indexInsertRecord(rec.value, .{ .off = rec.off, .len = rec.len }) catch break;
988. }
989. self.covered_bytes = @max(self.covered_bytes, file_size);
990. }
992. /// A file that already holds these exact bytes is left alone — its mtime
993. /// included — so closing a handle that changed nothing writes nothing.
994. fn fileHolds(alloc: std.mem.Allocator, path: []const u8, bytes: []const u8) bool {
995. const disk = fio.readAll(alloc, path) orelse return false;
996. defer alloc.free(disk);
997. return std.mem.eql(u8, disk, bytes);
998. }
1000. fn writeIndexFiles(self: *Db) Error!void {
1001. for (self.index_fields.items) |field| {
1002. const fi = self.idx.getPtr(field) orelse continue;
1003. var out: std.ArrayList(u8) = .empty;
1004. defer out.deinit(self.alloc);
1005. for (fi.entries.items) |e| {
1006. var kbuf: [32]u8 = undefined;
1007. const ks = switch (e.key) {
1008. .num => |n| jsNumFmt(&kbuf, n),
1009. .str => |s| s,
1010. };
1011. try self.appendFmt(&out, "{s},{d},{d}\n", .{ ks, e.off, e.len });
1012. }
1013. var pbuf: [512]u8 = undefined;
1014. const p = self.indexPath(&pbuf, field);
1015. if (fileHolds(self.alloc, p, out.items)) continue;
1016. var tbuf: [512]u8 = undefined;
1017. const tmp = std.fmt.bufPrint(&tbuf, "{s}.{d}.tmp", .{ p, linux.getpid() }) catch return Error.IoError;
1018. const fd = fio.openTrunc(self.alloc, tmp) orelse return Error.IoError;
1019. const ok = fio.writeAll(fd, out.items);
1020. fio.close(fd);
1021. if (!ok) return Error.IoError;
1022. if (!fio.rename(self.alloc, tmp, p)) return Error.IoError;
1023. }
1024. }
1026. fn readIdxState(self: *Db) ?u64 {
1027. const content = fio.readAll(self.alloc, self.idxstate_path) orelse return null;
1028. defer self.alloc.free(content);
1029. const parsed = std.json.parseFromSlice(std.json.Value, self.alloc, content, .{}) catch return null;
1030. defer parsed.deinit();
1031. if (parsed.value != .object) return null;
1032. const v = parsed.value.object.get("coveredBytes") orelse return null;
1033. return switch (v) {
1034. .integer => |i| @intCast(@max(i, 0)),
1035. .float => |f| @intFromFloat(@max(f, 0)),
1036. else => null,
1037. };
1038. }
1040. fn writeIdxState(self: *Db) Error!void {
1041. var buf: [128]u8 = undefined;
1042. const json = std.fmt.bufPrint(&buf, "{{\"coveredBytes\":{d}}}", .{self.covered_bytes}) catch return Error.IoError;
1043. if (fileHolds(self.alloc, self.idxstate_path, json)) return;
1044. var tbuf: [512]u8 = undefined;
1045. const tmp = std.fmt.bufPrint(&tbuf, "{s}.{d}.tmp", .{ self.idxstate_path, linux.getpid() }) catch return Error.IoError;
1046. const fd = fio.openTrunc(self.alloc, tmp) orelse return Error.IoError;
1047. const ok = fio.writeAll(fd, json);
1048. fio.close(fd);
1049. if (!ok) return Error.IoError;
1050. if (!fio.rename(self.alloc, tmp, self.idxstate_path)) return Error.IoError;
1051. }
1053. fn persistIndexesLocked(self: *Db) Error!void {
1054. try self.writeIndexFiles();
1055. // coverage can only grow (JS: max of ours and on-disk state)
1056. if (self.readIdxState()) |cb| self.covered_bytes = @max(self.covered_bytes, cb);
1057. try self.writeIdxState();
1058. }
1060. // =====================================================================
1061. // Operations
1062. // =====================================================================
1064. /// A mutable record under construction (op-arena entries).
1065. pub const MutableRecord = struct {
1066. entries: std.ArrayList(msgpack.Entry) = .empty,
1068. pub fn get(self: *const MutableRecord, key: []const u8) ?msgpack.Value {
1069. for (self.entries.items) |e| {
1070. if (e.key == .str and std.mem.eql(u8, e.key.str, key)) return e.value;
1071. }
1072. return null;
1073. }
1075. pub fn set(self: *MutableRecord, alloc: std.mem.Allocator, key: []const u8, v: msgpack.Value) Error!void {
1076. for (self.entries.items) |*e| {
1077. if (e.key == .str and std.mem.eql(u8, e.key.str, key)) {
1078. e.value = v;
1079. return;
1080. }
1081. }
1082. self.entries.append(alloc, .{ .key = .{ .str = key }, .value = v }) catch return Error.OutOfMemory;
1083. }
1085. pub fn toValue(self: *const MutableRecord) msgpack.Value {
1086. return .{ .map = self.entries.items };
1087. }
1088. };
1090. pub const InsertResult = union(enum) {
1091. pk_num: f64,
1092. pk_str: []const u8, // op-arena
1093. record: msgpack.Value,
1094. };
1096. /// insert() — auto primary key, unique checks, append, index.
1097. /// `record` must be a map value; `arena` is the op arena (record memory).
1098. pub fn insert(self: *Db, arena: std.mem.Allocator, record: msgpack.Value, skip_pk: bool) Error!InsertResult {
1099. if (record != .map) return Error.CorruptRecord;
1100. try self.acquireLock();
1101. defer self.releaseLock();
1102. self.refresh();
1103. return self.insertLocked(arena, record, skip_pk);
1104. }
1106. fn insertLocked(self: *Db, arena: std.mem.Allocator, record: msgpack.Value, skip_pk: bool) Error!InsertResult {
1107. // copy into mutable form
1108. var rec = MutableRecord{};
1109. for (record.map) |e| rec.entries.append(arena, e) catch return Error.OutOfMemory;
1111. // _hasPrimaryKeyValue: present and not null/undefined
1112. var has_pk_value = false;
1113. if (self.pk) |p| {
1114. if (rec.get(p)) |v| {
1115. has_pk_value = (v != .nil and v != .undef);
1116. }
1117. }
1119. var auto_gen = false;
1120. if (self.pk != null and !skip_pk and !has_pk_value) {
1121. if (self.pk_type == .number) {
1122. rec.set(arena, self.pk.?, .{ .number = self.next_id }) catch return Error.OutOfMemory;
1123. self.next_id += 1;
1124. auto_gen = true;
1125. } else if (self.pk_type == .uuid) {
1126. // A GENERATED ID IS CHECKED, NOT TRUSTED (ticket #113): the format is a
1127. // millisecond stamp + 3 base36 chars, so a bulk put collides within a
1128. // millisecond (two of 7800 in a measured run) and the unique check below
1129. // skips generated keys — two live rows then shared one pk and the next
1130. // update() deleted both and re-inserted one. Draw again until it is free.
1131. var ubuf: [12]u8 = undefined;
1132. var tries: usize = 0;
1133. while (true) : (tries += 1) {
1134. const u = genUuid(&ubuf);
1135. var taken: std.ArrayList(Loc) = .empty;
1136. defer taken.deinit(arena);
1137. try self.findPkLocs(arena, .{ .str = u }, &taken);
1138. if (taken.items.len == 0 or tries >= 64) break;
1139. }
1140. const owned = arena.dupe(u8, ubuf[0..12]) catch return Error.OutOfMemory;
1141. rec.set(arena, self.pk.?, .{ .str = owned }) catch return Error.OutOfMemory;
1142. auto_gen = true;
1143. }
1144. }
1146. // unique constraints (skip auto-generated pk)
1147. if (self.index_fields.items.len > 0 and self.unique_fields.items.len > 0) {
1148. var del_set = self.deletedSet(arena);
1149. for (self.unique_fields.items) |field| {
1150. if (auto_gen and self.pk != null and std.mem.eql(u8, field, self.pk.?)) continue;
1151. const v = rec.get(field) orelse continue;
1152. if (v == .undef) continue;
1153. const key = keyFromValue(v) orelse continue;
1154. var hits: std.ArrayList(Entry) = .empty;
1155. defer hits.deinit(arena);
1156. try self.indexGet(field, key, arena, &hits);
1157. var live: usize = 0;
1158. for (hits.items) |h| {
1159. if (!del_set.contains(h.off)) live += 1;
1160. }
1161. if (live > 0) {
1162. var kbuf: [32]u8 = undefined;
1163. const ks = switch (key) {
1164. .num => |n| jsNumFmt(&kbuf, n),
1165. .str => |s| s,
1166. };
1167. self.last_error = std.fmt.bufPrint(&self.last_error_buf, "Duplicate key: {s}={s}", .{ field, ks }) catch "Duplicate key";
1168. return Error.DuplicateKey;
1169. }
1170. }
1171. }
1173. // serialize + append
1174. const framed = msgpack.serialize(arena, rec.toValue()) catch return Error.OutOfMemory;
1175. const offset = fio.fileSize(self.alloc, self.data_path) orelse 0;
1176. const fd = fio.openAppend(self.alloc, self.data_path) orelse return Error.IoError;
1177. const ok = fio.writeAll(fd, framed);
1178. fio.close(fd);
1179. if (!ok) return Error.IoError;
1181. const loc = Loc{ .off = offset, .len = framed.len };
1182. if (self.index_fields.items.len > 0) {
1183. try self.indexInsertRecord(rec.toValue(), loc);
1184. }
1186. if (self.pk != null and self.pk_type == .number and !skip_pk and !has_pk_value) {
1187. try self.persistMetaLocked();
1188. }
1190. if (self.pk) |p| {
1191. const v = rec.get(p) orelse return Error.CorruptRecord;
1192. return switch (v) {
1193. .number => |n| InsertResult{ .pk_num = n },
1194. .str => |s| InsertResult{ .pk_str = s },
1195. else => InsertResult{ .record = rec.toValue() },
1196. };
1197. }
1198. return InsertResult{ .record = rec.toValue() };
1199. }
1201. /// Locate live records by primary key (index-backed when available).
1202. pub fn findPkLocs(self: *Db, arena: std.mem.Allocator, key: Key, out: *std.ArrayList(Loc)) Error!void {
1203. if (self.pk == null) return Error.NoPrimaryKey;
1204. self.refresh();
1205. var del_set = self.deletedSet(arena);
1206. if (self.index_fields.items.len > 0) {
1207. var hits: std.ArrayList(Entry) = .empty;
1208. defer hits.deinit(arena);
1209. try self.indexGet(self.pk.?, key, arena, &hits);
1210. for (hits.items) |h| {
1211. if (del_set.contains(h.off)) continue;
1212. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1213. }
1214. return;
1215. }
1216. // no indexes: full scan comparing the pk field
1217. var scanner = Scanner.init(self, arena, true);
1218. defer scanner.deinit();
1219. while (try scanner.next(arena)) |rec| {
1220. const v = rec.value.get(self.pk.?) orelse continue;
1221. const k = keyFromValue(v) orelse continue;
1222. if (cmpKeys(self.parseKeyForField(self.pk.?, k), self.parseKeyForField(self.pk.?, key)) == 0) {
1223. out.append(arena, .{ .off = rec.off, .len = rec.len }) catch return Error.OutOfMemory;
1224. }
1225. }
1226. }
1228. /// Locate live records by secondary index equality.
1229. pub fn findIndexLocs(self: *Db, arena: std.mem.Allocator, field: []const u8, key: Key, out: *std.ArrayList(Loc)) Error!void {
1230. self.refresh();
1231. var del_set = self.deletedSet(arena);
1232. var hits: std.ArrayList(Entry) = .empty;
1233. defer hits.deinit(arena);
1234. try self.indexGet(field, key, arena, &hits);
1235. for (hits.items) |h| {
1236. if (del_set.contains(h.off)) continue;
1237. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1238. }
1239. }
1241. /// Locate live records by index range [from, to] ascending.
1242. pub fn findRangeLocs(self: *Db, arena: std.mem.Allocator, field: []const u8, from: ?Key, to: ?Key, out: *std.ArrayList(Loc)) Error!void {
1243. self.refresh();
1244. var del_set = self.deletedSet(arena);
1245. var hits: std.ArrayList(Entry) = .empty;
1246. defer hits.deinit(arena);
1247. try self.indexRange(field, from, to, arena, &hits);
1248. var seen: std.AutoHashMapUnmanaged(u64, void) = .empty;
1249. for (hits.items) |h| {
1250. if (del_set.contains(h.off)) continue;
1251. if (seen.contains(h.off)) continue;
1252. seen.put(arena, h.off, {}) catch {};
1253. out.append(arena, .{ .off = h.off, .len = h.len }) catch return Error.OutOfMemory;
1254. }
1255. }
1257. /// delete by primary key. Returns the deleted records (decoded into arena).
1258. pub fn deletePk(self: *Db, arena: std.mem.Allocator, key: Key, out_records: *std.ArrayList(msgpack.Value)) Error!usize {
1259. try self.acquireLock();
1260. defer self.releaseLock();
1261. self.refresh();
1262. return self.deletePkLocked(arena, key, out_records);
1263. }
1265. fn deletePkLocked(self: *Db, arena: std.mem.Allocator, key: Key, out_records: *std.ArrayList(msgpack.Value)) Error!usize {
1266. var locs: std.ArrayList(Loc) = .empty;
1267. defer locs.deinit(arena);
1268. try self.findPkLocs(arena, key, &locs);
1269. for (locs.items) |loc| {
1270. const rec = try self.readAt(arena, loc);
1271. out_records.append(arena, rec) catch return Error.OutOfMemory;
1272. self.deleted.append(self.alloc, loc.off) catch return Error.OutOfMemory;
1273. self.indexRemoveOffset(loc.off);
1274. }
1275. try self.persistMetaLocked();
1276. return locs.items.len;
1277. }
1279. /// update by primary key: delete + insert(skip_pk) — JS update() semantics.
1280. /// `new_record` must already carry the primary key (the JS callback
1281. /// contract: the record keeps its pk unless the caller removes it).
1282. pub fn updatePk(self: *Db, arena: std.mem.Allocator, key: Key, new_record: msgpack.Value) Error!usize {
1283. if (self.pk) |p| {
1284. const carried = for (new_record.map) |e| {
1285. if (e.key == .str and std.mem.eql(u8, e.key.str, p)) break e.value != .nil and e.value != .undef;
1286. } else false;
1287. if (!carried) return Error.RecordLacksPrimaryKey;
1288. }
1289. try self.acquireLock();
1290. defer self.releaseLock();
1291. self.refresh();
1292. var old: std.ArrayList(msgpack.Value) = .empty;
1293. defer old.deinit(arena);
1294. const n = try self.deletePkLocked(arena, key, &old);
1295. if (n == 0) return 0;
1296. var i: usize = 0;
1297. while (i < n) : (i += 1) {
1298. _ = try self.insertLocked(arena, new_record, true);
1299. }
1300. return n;
1301. }
1303. /// compact() — rewrite the data file without tombstones, rebuild indexes.
1304. pub fn compact(self: *Db) Error!void {
1305. try self.acquireLock();
1306. defer self.releaseLock();
1307. self.refresh();
1308. try self.compactLocked();
1309. if (self.index_fields.items.len > 0) {
1310. try self.rebuildIndexes();
1311. }
1312. }
1314. fn compactLocked(self: *Db) Error!void {
1315. var scratch_state = std.heap.ArenaAllocator.init(self.arena_state.child_allocator);
1316. defer scratch_state.deinit();
1317. const scratch = scratch_state.allocator();
1319. var tbuf: [512]u8 = undefined;
1320. const tmp = std.fmt.bufPrint(&tbuf, "{s}.tmp", .{self.data_path}) catch return Error.IoError;
1321. const out_fd = fio.openTrunc(self.alloc, tmp) orelse return Error.IoError;
1323. var scanner = Scanner.init(self, scratch, true);
1324. var ok = true;
1325. while (true) {
1326. const maybe = scanner.nextRaw(scratch) catch {
1327. ok = false;
1328. break;
1329. };
1330. const raw = maybe orelse break;
1331. if (!fio.writeAll(out_fd, raw.bytes)) {
1332. ok = false;
1333. break;
1334. }
1335. }
1336. scanner.deinit();
1337. fio.close(out_fd);
1338. if (!ok) return Error.IoError;
1339. if (!fio.rename(self.alloc, tmp, self.data_path)) return Error.IoError;
1341. self.deleted.clearRetainingCapacity();
1342. try self.persistMetaLocked();
1343. }
1345. pub fn persistNow(self: *Db) Error!void {
1346. try self.acquireLock();
1347. defer self.releaseLock();
1348. self.refresh();
1349. try self.persistIndexesLocked();
1350. }
1351. };
1353. // =========================================================================
1354. // Tests
1355. // =========================================================================
1357. const testing = std.testing;
1359. fn tmpBase(buf: []u8, comptime name: []const u8) []const u8 {
1360. return std.fmt.bufPrint(buf, "/tmp/mpackdb-zigtest-{d}-" ++ name, .{linux.getpid()}) catch unreachable;
1361. }
1363. fn cleanup(alloc: std.mem.Allocator, base: []const u8) void {
1364. var buf: [512]u8 = undefined;
1365. const suffixes = [_][]const u8{ ".mpack", ".meta.json", ".idxstate.json", ".lock", ".id.txt", ".email.txt", ".age.txt", ".uuid.txt" };
1366. for (suffixes) |suffix| {
1367. const p = std.fmt.bufPrint(&buf, "{s}{s}", .{ base, suffix }) catch continue;
1368. _ = fio.unlink(alloc, p);
1369. }
1370. }
1372. fn strKey(s: []const u8) Key {
1373. return .{ .str = s };
1374. }
1376. fn numKey(n: f64) Key {
1377. return .{ .num = n };
1378. }
1380. fn makeUser(arena: std.mem.Allocator, name: []const u8, email: []const u8, age: f64) !msgpack.Value {
1381. const entries = try arena.alloc(msgpack.Entry, 3);
1382. entries[0] = .{ .key = .{ .str = "name" }, .value = .{ .str = name } };
1383. entries[1] = .{ .key = .{ .str = "email" }, .value = .{ .str = email } };
1384. entries[2] = .{ .key = .{ .str = "age" }, .value = .{ .number = age } };
1385. return .{ .map = entries };
1386. }
1388. test "engine: insert/find/delete/update with numeric pk + indexes" {
1389. var base_buf: [128]u8 = undefined;
1390. const base = tmpBase(&base_buf, "crud");
1391. cleanup(testing.allocator, base);
1392. defer cleanup(testing.allocator, base);
1394. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1395. defer arena_state.deinit();
1396. const arena = arena_state.allocator();
1398. const db = try Db.open(testing.allocator, base, .{
1399. .primary_key = "*id",
1400. .indexes = &.{ "email", "*age" },
1401. });
1403. // insert three — a fresh table counts from 1 (ticket #4)
1404. const r1 = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 30), false);
1405. try testing.expectEqual(@as(f64, 1), r1.pk_num);
1406. const r2 = try db.insert(arena, try makeUser(arena, "Bob", "[email protected]", 25), false);
1407. try testing.expectEqual(@as(f64, 2), r2.pk_num);
1408. _ = try db.insert(arena, try makeUser(arena, "Carol", "[email protected]", 35), false);
1410. // find by pk
1411. var locs: std.ArrayList(Loc) = .empty;
1412. try db.findPkLocs(arena, numKey(2), &locs);
1413. try testing.expectEqual(@as(usize, 1), locs.items.len);
1414. const bob = try db.readAt(arena, locs.items[0]);
1415. try testing.expectEqualStrings("Bob", bob.get("name").?.str);
1417. // find by secondary index
1418. var locs2: std.ArrayList(Loc) = .empty;
1419. try db.findIndexLocs(arena, "email", strKey("[email protected]"), &locs2);
1420. try testing.expectEqual(@as(usize, 1), locs2.items.len);
1422. // range on numeric index: age 26..40 → Alice(30), Carol(35)
1423. var locs3: std.ArrayList(Loc) = .empty;
1424. try db.findRangeLocs(arena, "age", numKey(26), numKey(40), &locs3);
1425. try testing.expectEqual(@as(usize, 2), locs3.items.len);
1427. // update Bob's age
1428. var bob_new = Db.MutableRecord{};
1429. for (bob.map) |e| try bob_new.entries.append(arena, e);
1430. try bob_new.set(arena, "age", .{ .number = 26 });
1431. const updated = try db.updatePk(arena, numKey(2), bob_new.toValue());
1432. try testing.expectEqual(@as(usize, 1), updated);
1434. var locs4: std.ArrayList(Loc) = .empty;
1435. try db.findPkLocs(arena, numKey(2), &locs4);
1436. try testing.expectEqual(@as(usize, 1), locs4.items.len);
1437. const bob2 = try db.readAt(arena, locs4.items[0]);
1438. try testing.expectEqual(@as(f64, 26), bob2.get("age").?.number);
1440. // delete Alice
1441. var deleted_recs: std.ArrayList(msgpack.Value) = .empty;
1442. const dn = try db.deletePk(arena, numKey(1), &deleted_recs);
1443. try testing.expectEqual(@as(usize, 1), dn);
1444. try testing.expectEqualStrings("Alice", deleted_recs.items[0].get("name").?.str);
1446. var locs5: std.ArrayList(Loc) = .empty;
1447. try db.findPkLocs(arena, numKey(1), &locs5);
1448. try testing.expectEqual(@as(usize, 0), locs5.items.len);
1450. db.close();
1452. // reopen (compaction drops the tombstone) and verify persistence
1453. const db2 = try Db.open(testing.allocator, base, .{
1454. .primary_key = "*id",
1455. .indexes = &.{ "email", "*age" },
1456. });
1457. defer db2.close();
1458. var locs6: std.ArrayList(Loc) = .empty;
1459. try db2.findPkLocs(arena, numKey(2), &locs6);
1460. try testing.expectEqual(@as(usize, 1), locs6.items.len);
1461. const bob3 = try db2.readAt(arena, locs6.items[0]);
1462. try testing.expectEqual(@as(f64, 26), bob3.get("age").?.number);
1463. // nextId continues after reopen
1464. const r4 = try db2.insert(arena, try makeUser(arena, "Dan", "[email protected]", 40), false);
1465. try testing.expectEqual(@as(f64, 4), r4.pk_num);
1466. }
1468. test "engine: a stored counter wins over FIRST_ID (ticket #4)" {
1469. var base_buf: [128]u8 = undefined;
1470. const base = tmpBase(&base_buf, "counter");
1471. cleanup(testing.allocator, base);
1472. defer cleanup(testing.allocator, base);
1474. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1475. defer arena_state.deinit();
1476. const arena = arena_state.allocator();
1478. // A table the JS mpackdb created and never wrote to: its meta says 0, and
1479. // an existing table keeps its own counter.
1480. var mbuf: [160]u8 = undefined;
1481. const meta_path = try std.fmt.bufPrint(&mbuf, "{s}.meta.json", .{base});
1482. const fd = fio.openTrunc(testing.allocator, meta_path).?;
1483. try testing.expect(fio.writeAll(fd, "{\"nextId\":0,\"deleted\":[],\"version\":1}"));
1484. fio.close(fd);
1486. const db = try Db.open(testing.allocator, base, .{ .primary_key = "*id" });
1487. defer db.close();
1488. const r = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 1), false);
1489. try testing.expectEqual(@as(f64, 0), r.pk_num);
1490. }
1492. test "engine: unique index rejects duplicates" {
1493. var base_buf: [128]u8 = undefined;
1494. const base = tmpBase(&base_buf, "uniq");
1495. cleanup(testing.allocator, base);
1496. defer cleanup(testing.allocator, base);
1498. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1499. defer arena_state.deinit();
1500. const arena = arena_state.allocator();
1502. const db = try Db.open(testing.allocator, base, .{
1503. .primary_key = "*id",
1504. .indexes = &.{"!email"},
1505. });
1506. defer db.close();
1508. _ = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 30), false);
1509. const dup = db.insert(arena, try makeUser(arena, "Evil", "[email protected]", 31), false);
1510. try testing.expectError(Error.DuplicateKey, dup);
1511. try testing.expect(std.mem.indexOf(u8, db.last_error, "[email protected]") != null);
1512. }
1514. test "engine: uuid primary key" {
1515. var base_buf: [128]u8 = undefined;
1516. const base = tmpBase(&base_buf, "uuid");
1517. cleanup(testing.allocator, base);
1518. defer cleanup(testing.allocator, base);
1520. var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
1521. defer arena_state.deinit();
1522. const arena = arena_state.allocator();
1524. const db = try Db.open(testing.allocator, base, .{ .primary_key = "@uuid" });
1525. defer db.close();
1527. const r = try db.insert(arena, try makeUser(arena, "Alice", "[email protected]", 1), false);
1528. try testing.expectEqual(@as(usize, 12), r.pk_str.len);
1529. var locs: std.ArrayList(Loc) = .empty;
1530. try db.findPkLocs(arena, strKey(r.pk_str), &locs);
1531. try testing.expectEqual(@as(usize, 1), locs.items.len);
1532. }
1534. test "jsNumFmt / jsParseInt / uuid shape" {
1535. var buf: [32]u8 = undefined;
1536. try testing.expectEqualStrings("42", jsNumFmt(&buf, 42));
1537. try testing.expectEqualStrings("-7", jsNumFmt(&buf, -7));
1538. try testing.expectEqualStrings("1.5", jsNumFmt(&buf, 1.5));
1539. try testing.expectEqual(@as(f64, 1), jsParseInt("1.5").?);
1540. try testing.expectEqual(@as(f64, -12), jsParseInt("-12abc").?);
1541. try testing.expect(jsParseInt("abc") == null);
1542. var ubuf: [12]u8 = undefined;
1543. const u = genUuid(&ubuf);
1544. try testing.expectEqual(@as(usize, 12), u.len);
1545. for (u) |ch| try testing.expect((ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'z'));
1546. }