commit 13ee0a66101f41787171d45d3f9f61e68200ee9c
parent ddd3faabc02a028b253014b1f0025e8410ba74f1
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Fri, 18 Sep 2026 10:34:37 -0400
ntfs: read the MFT in parallel
The scan spent two thirds of its time waiting on one read at a time. Chunks
of the read plan now go through flow.each, each worker holding its own
handle, buffer, arena and stats. The only shared structure is the entry
table, and every write lands at a record number no other worker claims.
Extension records are the exception, crediting a base record another worker
may own, so they are copied aside and folded in serially afterwards.
Four readers take 1357 ms against 1999 for one. The drive saturates almost at
once, so more is worse: thirty-two takes 2339 ms.
Diffstat:
| M | ntfs/mft.odin | | | 95 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------- |
| M | ntfs/ntfs_test.odin | | | 21 | +++++++++++---------- |
| M | ntfs/reader.odin | | | 250 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------- |
3 files changed, 284 insertions(+), 82 deletions(-)
diff --git a/ntfs/mft.odin b/ntfs/mft.odin
@@ -45,23 +45,50 @@ Mft_Stats :: struct {
planned_bytes: u64,
skipped_bytes: u64,
extents: u64,
+ workers: u64, // threads the scan actually ran on
// Phase timings, nanoseconds.
io_ns: i64,
parse_ns: i64,
bitmap_ns: i64,
}
+/*
+A record that belongs to a base record elsewhere in the table.
+
+Its attributes are credited to the base entry, which another worker may own, so it
+cannot be folded in during a parallel pass. The bytes are copied because the read
+buffer is reused for the next chunk.
+*/
+Deferred :: struct {
+ record: u32,
+ bytes: []byte,
+}
+
+/*
+Where one worker puts what it produces.
+
+Entries are written straight into the table at their own record numbers, which no
+two workers share, so only these need merging afterwards. The arena outlives the run
+because every `Entry.name` points into it.
+*/
+Sink :: struct {
+ names: virtual.Arena,
+ links: [dynamic]Hard_Link,
+ deferred: [dynamic]Deferred,
+ stats: Mft_Stats,
+}
+
Mft :: struct {
entries: []Entry,
links: [dynamic]Hard_Link,
+ sinks: []Sink, // one per worker; kept alive for the names they hold
boot: Boot_Sector,
bytes_per_cluster: u64,
stats: Mft_Stats,
- names: virtual.Arena, // backing store for every name string; never moves
allocator: mem.Allocator,
}
-mft_init :: proc(m: ^Mft, record_count: int, bytes_per_cluster: u64, allocator := context.allocator) -> Error {
+mft_init :: proc(m: ^Mft, record_count: int, bytes_per_cluster: u64, sinks := 1, allocator := context.allocator) -> Error {
m.allocator = allocator
m.bytes_per_cluster = bytes_per_cluster
entries, err := make([]Entry, record_count, allocator)
@@ -70,10 +97,14 @@ mft_init :: proc(m: ^Mft, record_count: int, bytes_per_cluster: u64, allocator :
}
m.entries = entries
m.links = make([dynamic]Hard_Link, allocator)
- if virtual.arena_init_growing(&m.names) != nil {
- delete(m.entries, allocator)
- delete(m.links)
- return .Out_Of_Memory
+ m.sinks = make([]Sink, max(sinks, 1), allocator)
+ for &s in m.sinks {
+ if virtual.arena_init_growing(&s.names) != nil {
+ mft_destroy(m)
+ return .Out_Of_Memory
+ }
+ s.links.allocator = allocator
+ s.deferred.allocator = allocator
}
m.stats.records = u64(record_count)
return .None
@@ -82,10 +113,34 @@ mft_init :: proc(m: ^Mft, record_count: int, bytes_per_cluster: u64, allocator :
mft_destroy :: proc(m: ^Mft) {
delete(m.entries, m.allocator)
delete(m.links)
- virtual.arena_destroy(&m.names)
+ for &s in m.sinks {
+ delete(s.links)
+ delete(s.deferred)
+ virtual.arena_destroy(&s.names)
+ }
+ delete(m.sinks, m.allocator)
m^ = {}
}
+// Fold each worker's private results into the table. Only these needed merging;
+// entries were written into slots no other worker could reach.
+mft_merge_sinks :: proc(m: ^Mft) {
+ for &s in m.sinks {
+ m.stats.records_read += s.stats.records_read
+ m.stats.records_bad += s.stats.records_bad
+ m.stats.in_use += s.stats.in_use
+ m.stats.directories += s.stats.directories
+ m.stats.resident_files += s.stats.resident_files
+ m.stats.resident_bytes += s.stats.resident_bytes
+ m.stats.io_ns += s.stats.io_ns
+ m.stats.parse_ns += s.stats.parse_ns
+ for l in s.links {
+ append(&m.links, l)
+ }
+ clear(&s.links)
+ }
+}
+
/*
Fold one fixed-up FILE record into the table. Records may arrive in any order.
@@ -98,7 +153,7 @@ attribute anyway.
Records not in use are skipped entirely. Their contents are stale and may point at
records that have since been reused.
*/
-mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte) -> Error {
+mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte, sink: ^Sink) -> Error {
if len(rec) < size_of(Record_Header) {
return .Bad_Record
}
@@ -125,9 +180,9 @@ mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte) -> Error {
e.sequence = h.sequence
e.link_count = h.link_count
e.flags = h.flags
- m.stats.in_use += 1
+ sink.stats.in_use += 1
if .Directory in h.flags {
- m.stats.directories += 1
+ sink.stats.directories += 1
}
}
@@ -144,7 +199,7 @@ mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte) -> Error {
}
case .File_Name:
if fn, fn_ok := file_name(a); fn_ok {
- add_name(m, owner, fn)
+ add_name(m, owner, fn, sink)
}
case .Data:
if a.non_resident {
@@ -156,8 +211,8 @@ mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte) -> Error {
}
} else if len(a.name) == 0 {
e.size = u64(len(a.value))
- m.stats.resident_files += 1
- m.stats.resident_bytes += u64(len(a.value))
+ sink.stats.resident_files += 1
+ sink.stats.resident_bytes += u64(len(a.value))
}
case:
// Directory indexes, bitmaps, reparse data, and EFS streams occupy clusters too.
@@ -175,13 +230,13 @@ pointing at the same parent. The alias is not a separate link, so it is skipped
whenever a proper name exists. Any further Win32 or POSIX name is a real hard link.
*/
@(private)
-add_name :: proc(m: ^Mft, owner: u32, fn: File_Name) {
+add_name :: proc(m: ^Mft, owner: u32, fn: File_Name, sink: ^Sink) {
e := &m.entries[owner]
parent := u32(ref_record(fn.parent))
parent_seq := ref_sequence(fn.parent)
if e.name == "" {
- e.name = intern_utf16(m, fn.name)
+ e.name = intern_utf16(sink, fn.name)
e.parent = parent
e.parent_sequence = parent_seq
e.namespace = fn.namespace
@@ -192,29 +247,29 @@ add_name :: proc(m: ^Mft, owner: u32, fn: File_Name) {
}
if e.namespace == .Dos {
// The alias arrived first; the real name replaces it.
- e.name = intern_utf16(m, fn.name)
+ e.name = intern_utf16(sink, fn.name)
e.parent = parent
e.parent_sequence = parent_seq
e.namespace = fn.namespace
return
}
- name := intern_utf16(m, fn.name)
+ name := intern_utf16(sink, fn.name)
if name == e.name && parent == e.parent {
return
}
- append(&m.links, Hard_Link{record = owner, parent = parent, name = name})
+ append(&sink.links, Hard_Link{record = owner, parent = parent, name = name})
}
// Copy a UTF-16LE name into the arena as UTF-8. Names are at most 255 code units.
@(private)
-intern_utf16 :: proc(m: ^Mft, name: []u16) -> string {
+intern_utf16 :: proc(sink: ^Sink, name: []u16) -> string {
runes: [256]rune
n := utf16.decode(runes[:], name)
total := 0
for r in runes[:n] {
total += utf8.rune_size(r)
}
- buf, err := make([]byte, total, virtual.arena_allocator(&m.names))
+ buf, err := make([]byte, total, virtual.arena_allocator(&sink.names))
if err != nil {
return ""
}
diff --git a/ntfs/ntfs_test.odin b/ntfs/ntfs_test.odin
@@ -176,7 +176,8 @@ build_record :: proc(
@(private = "file")
add :: proc(t: ^testing.T, m: ^Mft, rec: []byte) {
testing.expect_value(t, apply_fixups(rec), Error.None)
- testing.expect_value(t, mft_add_record(m, record_header(rec).record_number, rec), Error.None)
+ testing.expect_value(t, mft_add_record(m, record_header(rec).record_number, rec, &m.sinks[0]), Error.None)
+ mft_merge_sinks(m)
}
// ---- boot sector --------------------------------------------------------------------
@@ -273,7 +274,7 @@ test_resident_file :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
@@ -302,7 +303,7 @@ test_dos_name_first_is_replaced :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].name, "long name.txt")
@@ -325,7 +326,7 @@ test_nonresident_streams :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
@@ -347,7 +348,7 @@ test_fragmented_extents_sum_their_own_clusters :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].allocated, u64(8192))
@@ -366,7 +367,7 @@ test_unallocated_runlist_counts_nothing :: proc(t: ^testing.T) {
record_number = RECORD_BAD_CLUS,
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[RECORD_BAD_CLUS].allocated, u64(0))
@@ -398,7 +399,7 @@ test_extension_record_and_hard_links :: proc(t: ^testing.T) {
base = make_ref(100, 9),
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
// The extension arrives before its base, as it can on disk.
@@ -429,7 +430,7 @@ test_free_record_is_ignored :: proc(t: ^testing.T) {
flags = {},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].name, "")
@@ -441,7 +442,7 @@ test_free_record_is_ignored :: proc(t: ^testing.T) {
@(test)
test_path :: proc(t: ^testing.T) {
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
root := make_ref(RECORD_ROOT, 5)
@@ -505,7 +506,7 @@ test_resident_file_costs_no_clusters :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096, 1), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
diff --git a/ntfs/reader.odin b/ntfs/reader.odin
@@ -1,14 +1,33 @@
package ntfs
import "core:mem"
+import "core:mem/virtual"
import "core:time"
+import "../flow"
+
// Alignment for I/O buffers. Raw volume reads want sector alignment; 4 KiB covers
// every sector size in use.
@(private)
IO_ALIGN :: 4096
-DEFAULT_CHUNK_SIZE :: 16 * mem.Megabyte
+// Bytes per read. Smaller than a serial reader would want: queue depth comes from
+// the number of workers, and smaller chunks divide more evenly between them.
+DEFAULT_CHUNK_SIZE :: 4 * mem.Megabyte
+
+/*
+Readers to run concurrently.
+
+Measured by sweeping this on a live NVMe volume: one reader takes 1807 ms, two take
+1338, and four through eight all sit at about 1310. Past that it degrades, and
+thirty-two takes 2339 ms, worse than reading serially. Aggregate read time grows in
+step with the count throughout, which is what a device already at its limit looks
+like: extra readers divide the same bandwidth rather than adding to it.
+
+So this is not a function of the core count. Four sits in the flat part of the curve
+with margin for a device that does want more depth.
+*/
+DEFAULT_WORKERS :: 4
/*
How volume reads reach memory.
@@ -43,6 +62,7 @@ Read_Options :: struct {
chunk_size: int, // bytes per volume read; 0 selects DEFAULT_CHUNK_SIZE
io_mode: IO_Mode,
min_skip: int, // 0 selects DEFAULT_MIN_SKIP; negative reads the whole table
+ workers: int, // 0 selects DEFAULT_WORKERS
}
// Open `path` (drive letter or image file) and read its whole MFT into a table.
@@ -156,18 +176,7 @@ read_mft_from_volume :: proc(v: ^Volume, opts := Read_Options{}, allocator := co
return {}, .Mft_Data_Missing
}
- record_count := int(mft_bytes / u64(record_size))
- if init_err := mft_init(&m, record_count, cluster, allocator); init_err != nil {
- return {}, init_err
- }
- m.boot = boot
- // From here on, failures must release the table.
- ok := false
- defer if !ok {
- mft_destroy(&m)
- }
-
- // 3. Stream the table.
+ // 3. Plan what to read, and cut it into chunks small enough to hand out.
chunk := opts.chunk_size
if chunk <= 0 {
chunk = DEFAULT_CHUNK_SIZE
@@ -175,14 +184,8 @@ read_mft_from_volume :: proc(v: ^Volume, opts := Read_Options{}, allocator := co
// A chunk must hold whole records and whole clusters so every read stays aligned.
unit := max(record_size, int(cluster))
chunk = max(chunk / unit, 1) * unit
- buf, buf_alloc_err := mem.alloc_bytes(chunk, IO_ALIGN, allocator)
- if buf_alloc_err != nil {
- return {}, .Out_Of_Memory
- }
- defer mem.free_bytes(buf, allocator)
- // Decide what to read. Without the slot bitmap, or with skipping turned off, the
- // plan is the whole table, which is what the reader did before.
+ // Without the slot bitmap, or with skipping turned off, the plan is the whole table.
extents: []Read_Extent
defer delete(extents, allocator)
if slot_bitmap != nil && opts.min_skip >= 0 {
@@ -202,57 +205,200 @@ read_mft_from_volume :: proc(v: ^Volume, opts := Read_Options{}, allocator := co
whole[0] = Read_Extent{0, mft_bytes}
extents = whole
}
+
+ chunks := make([dynamic]Chunk, allocator)
+ defer delete(chunks)
+ planned_bytes: u64
for e in extents {
- m.stats.planned_bytes += e.length
+ planned_bytes += e.length
+ for offset := e.offset; offset < e.offset + e.length; offset += u64(chunk) {
+ append(&chunks, Chunk{offset, min(u64(chunk), e.offset + e.length - offset)})
+ }
+ }
+
+ // 4. Size the pool. The limit is the measured one, not the machine's, because the
+ // drive saturates long before the cores do.
+ worker_count := flow.width(len(chunks), .Io, limit = opts.workers if opts.workers > 0 else DEFAULT_WORKERS)
+
+ record_count := int(mft_bytes / u64(record_size))
+ if init_err := mft_init(&m, record_count, cluster, worker_count, allocator); init_err != nil {
+ return {}, init_err
}
- m.stats.skipped_bytes = mft_bytes - m.stats.planned_bytes
+ m.boot = boot
+ // From here on, failures must release the table.
+ ok := false
+ defer if !ok {
+ mft_destroy(&m)
+ }
+ m.stats.planned_bytes = planned_bytes
+ m.stats.skipped_bytes = mft_bytes - planned_bytes
m.stats.extents = u64(len(extents))
- reader := Extent_Reader{v = v, runs = runs, cluster = cluster}
- for e in extents {
- for offset := e.offset; offset < e.offset + e.length; offset += u64(chunk) {
- n := int(min(u64(chunk), e.offset + e.length - offset))
- // Unbuffered reads must cover whole sectors. The run list is cluster
- // granular, so rounding up never reads past the attribute's allocation.
- read_n := min(int((u64(n) + cluster - 1) / cluster * cluster), chunk)
- io_start := time.tick_now()
- if read_err := read_logical(&reader, offset, buf[:read_n]); read_err != nil {
- return {}, read_err
+ workers, workers_err := make([]Worker, worker_count, allocator)
+ if workers_err != nil {
+ return {}, .Out_Of_Memory
+ }
+ defer delete(workers, allocator)
+
+ // Windows serialises I/O on one handle, so a worker that shares it would queue
+ // behind its neighbours instead of adding queue depth. Each takes its own.
+ live := 0
+ for i in 0 ..< worker_count {
+ w := &workers[i]
+ w.mft = &m
+ w.sink = &m.sinks[i]
+ w.record_size = record_size
+ w.cluster = cluster
+ wv, clone_err := volume_clone(v, allocator)
+ if clone_err != nil {
+ break
+ }
+ wbuf, buf_err := mem.alloc_bytes(chunk, IO_ALIGN, allocator)
+ if buf_err != nil {
+ volume_close(&wv)
+ break
+ }
+ w.volume = wv
+ w.owns_volume = true
+ w.buf = wbuf
+ w.reader = Extent_Reader{v = &w.volume, runs = runs, cluster = cluster}
+ live += 1
+ }
+ if live == 0 {
+ // No second handle to be had, so fall back to reading on the caller's.
+ w := &workers[0]
+ wbuf, buf_err := mem.alloc_bytes(chunk, IO_ALIGN, allocator)
+ if buf_err != nil {
+ return {}, .Out_Of_Memory
+ }
+ w.volume = v^
+ w.buf = wbuf
+ w.reader = Extent_Reader{v = &w.volume, runs = runs, cluster = cluster}
+ live = 1
+ }
+ defer {
+ for i in 0 ..< live {
+ w := &workers[i]
+ if w.owns_volume {
+ volume_close(&w.volume)
}
- m.stats.io_ns += i64(time.tick_since(io_start))
- parse_start := time.tick_now()
- defer m.stats.parse_ns += i64(time.tick_since(parse_start))
- for start := 0; start + record_size <= n; start += record_size {
- rec := buf[start:start + record_size]
- // Extents may skip forward, so the slot number comes from the offset.
- record_number := u32((offset + u64(start)) / u64(record_size))
- // Slots never written are all zeros; skip them silently.
- if rd32(rec, 0) != RECORD_MAGIC {
- continue
- }
- m.stats.records_read += 1
- if apply_fixups(rec) != nil {
- m.stats.records_bad += 1
- continue
- }
- if add_err := mft_add_record(&m, record_number, rec); add_err != nil {
- m.stats.records_bad += 1
- }
+ mem.free_bytes(w.buf, allocator)
+ }
+ }
+ m.stats.workers = u64(live)
+
+ // 5. Read and fold, every worker on its own chunk.
+ flow.each(chunks[:], workers[:live], read_chunk, .Mixed)
+ for &w in workers[:live] {
+ if w.err != nil {
+ return {}, w.err
+ }
+ }
+
+ // 6. Extension records were held back because they credit a base record another
+ // worker may own. Every base record is in place now, so they fold in serially.
+ for &w in workers[:live] {
+ for d in w.sink.deferred {
+ if add_err := mft_add_record(&m, d.record, d.bytes, w.sink); add_err != nil {
+ w.sink.stats.records_bad += 1
}
}
+ clear(&w.sink.deferred)
}
+
// $Bitmap is the file system's own count of used clusters, which checks the sums
// built above. Failing to read it costs nothing else, so the table still stands.
bitmap_start := time.tick_now()
- if c, bitmap_err := read_bitmap_clusters(&reader, boot, buf, allocator); bitmap_err == nil {
+ if c, bitmap_err := read_bitmap_clusters(&workers[0].reader, boot, workers[0].buf, allocator); bitmap_err == nil {
m.stats.allocated_clusters = c
}
m.stats.bitmap_ns = i64(time.tick_since(bitmap_start))
+ mft_merge_sinks(&m)
ok = true
return m, .None
}
+// A slice of the MFT's logical address space small enough to be one read, and the
+// unit of parallel work: one worker claims it, reads it, and folds it in.
+@(private)
+Chunk :: struct {
+ offset: u64,
+ length: u64,
+}
+
+/*
+One worker's private world for a parallel scan.
+
+Everything here belongs to a single worker except `mft`, and every write into that
+lands at a record number no other worker claims. Extension records are the exception
+and are held in the sink rather than written across the divide.
+*/
+@(private)
+Worker :: struct {
+ mft: ^Mft,
+ sink: ^Sink,
+ volume: Volume,
+ owns_volume: bool,
+ reader: Extent_Reader,
+ buf: []byte,
+ record_size: int,
+ cluster: u64,
+ err: Error,
+}
+
+@(private)
+read_chunk :: proc(c: Chunk, w: ^Worker) -> bool {
+ n := int(c.length)
+ // Unbuffered reads must cover whole sectors. The run list is cluster granular, so
+ // rounding up never reads past the attribute's allocation.
+ read_n := min(int((c.length + w.cluster - 1) / w.cluster * w.cluster), len(w.buf))
+ io_start := time.tick_now()
+ if read_err := read_logical(&w.reader, c.offset, w.buf[:read_n]); read_err != nil {
+ w.err = read_err
+ return false
+ }
+ w.sink.stats.io_ns += i64(time.tick_since(io_start))
+
+ parse_start := time.tick_now()
+ defer w.sink.stats.parse_ns += i64(time.tick_since(parse_start))
+ for start := 0; start + w.record_size <= n; start += w.record_size {
+ rec := w.buf[start:start + w.record_size]
+ // Chunks skip forward, so the slot number comes from the offset.
+ record := u32((c.offset + u64(start)) / u64(w.record_size))
+ // Slots never written are all zeros; skip them silently.
+ if rd32(rec, 0) != RECORD_MAGIC {
+ continue
+ }
+ w.sink.stats.records_read += 1
+ if apply_fixups(rec) != nil {
+ w.sink.stats.records_bad += 1
+ continue
+ }
+ if record_header(rec).base_record != 0 {
+ hold_extension_record(w, record, rec)
+ continue
+ }
+ if add_err := mft_add_record(w.mft, record, rec, w.sink); add_err != nil {
+ w.sink.stats.records_bad += 1
+ }
+ }
+ return true
+}
+
+// Copy an extension record into the sink's arena, since the buffer is about to be
+// reused for the next chunk.
+@(private)
+hold_extension_record :: proc(w: ^Worker, record: u32, rec: []byte) {
+ bytes, err := make([]byte, len(rec), virtual.arena_allocator(&w.sink.names))
+ if err != nil {
+ w.sink.stats.records_bad += 1
+ return
+ }
+ copy(bytes, rec)
+ append(&w.sink.deferred, Deferred{record = record, bytes = bytes})
+}
+
// Read a whole attribute into memory: a copy of the value when resident, or the
// clusters its run list names when not. The caller frees the result.
@(private)
