commit d66668cf8754a802792ffc92319889d3f5148f0a
parent 3bff5ee1bb8fdb6edd17ae05d2c12cfcf4f3d1ee
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Wed, 16 Sep 2026 21:18:31 -0400
ntfs: measure allocation from run lists instead of header sizes
A live scan reported $BadClus at 491 GiB and the root directory at 942 GiB
on a 491 GiB volume. Its $Bad stream spans the volume with entirely
unallocated runs yet carries no sparse flag, so allocated_size is fiction.
The run list is the ground truth: summing its non-sparse clusters is correct
for plain, compressed, and sparse attributes alike and needs no flag checks.
Each extent maps only its own VCN range, so every extent of a fragmented
attribute now contributes, with the header sizes kept as a fallback for a
malformed list. mft_init takes the cluster size to do the multiplication.
Diffstat:
5 files changed, 109 insertions(+), 34 deletions(-)
diff --git a/ntfs/mft.odin b/ntfs/mft.odin
@@ -37,16 +37,18 @@ Mft_Stats :: struct {
}
Mft :: struct {
- entries: []Entry,
- links: [dynamic]Hard_Link,
- boot: Boot_Sector,
- stats: Mft_Stats,
- names: virtual.Arena, // backing store for every name string; never moves
- allocator: mem.Allocator,
+ entries: []Entry,
+ links: [dynamic]Hard_Link,
+ boot: Boot_Sector,
+ bytes_per_cluster: u64,
+ stats: Mft_Stats,
+ names: virtual.Arena, // backing store for every name string; never moves
+ allocator: mem.Allocator,
}
-mft_init :: proc(m: ^Mft, record_count: int, allocator := context.allocator) -> Error {
+mft_init :: proc(m: ^Mft, record_count: int, bytes_per_cluster: u64, allocator := context.allocator) -> Error {
m.allocator = allocator
+ m.bytes_per_cluster = bytes_per_cluster
entries, err := make([]Entry, record_count, allocator)
if err != nil {
return .Out_Of_Memory
@@ -131,21 +133,18 @@ mft_add_record :: proc(m: ^Mft, record_number: u32, rec: []byte) -> Error {
}
case .Data:
if a.non_resident {
- // Only the first extent carries the sizes; later extents repeat them.
- if a.lowest_vcn == 0 {
- e.allocated += attr_disk_size(a)
- if len(a.name) == 0 {
- e.size = a.data_size
- }
+ // Every extent contributes its own clusters, but only the first carries the
+ // logical size; later extents repeat it.
+ e.allocated += attr_disk_size(a, m.bytes_per_cluster)
+ if a.lowest_vcn == 0 && len(a.name) == 0 {
+ e.size = a.data_size
}
} else if len(a.name) == 0 {
e.size = u64(len(a.value))
}
case:
// Directory indexes, bitmaps, reparse data, and EFS streams occupy clusters too.
- if a.non_resident && a.lowest_vcn == 0 {
- e.allocated += attr_disk_size(a)
- }
+ e.allocated += attr_disk_size(a, m.bytes_per_cluster)
}
}
return .None
diff --git a/ntfs/ntfs_test.odin b/ntfs/ntfs_test.odin
@@ -273,7 +273,7 @@ test_resident_file :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
@@ -302,7 +302,7 @@ test_dos_name_first_is_replaced :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].name, "long name.txt")
@@ -311,29 +311,34 @@ test_dos_name_first_is_replaced :: proc(t: ^testing.T) {
@(test)
test_nonresident_streams :: proc(t: ^testing.T) {
- runlist := []byte{0x11, 0x02, 0x10, 0x00}
+ two_clusters := []byte{0x11, 0x02, 0x10, 0x00}
+ one_cluster := []byte{0x11, 0x01, 0x20, 0x00}
+ // 16 allocated clusters followed by 240 sparse ones: a 1 MiB compressed stream
+ // that squeezed into 64 KiB.
+ compressed := []byte{0x11, 0x10, 0x30, 0x01, 0xF0, 0x00}
rec := build_record(
{
resident(.File_Name, file_name_value(make_ref(64, 3), "big.bin", .Win32_And_Dos)),
- nonresident(.Data, 8192, 5000, runlist),
- nonresident(.Data, 4096, 100, runlist, name = "Zone.Identifier"),
- nonresident(.Data, 1 << 20, 1 << 20, runlist, name = "packed", flags = {.Compressed}, compressed_size = 65536),
+ nonresident(.Data, 8192, 5000, two_clusters),
+ nonresident(.Data, 4096, 100, one_cluster, name = "Zone.Identifier"),
+ nonresident(.Data, 1 << 20, 1 << 20, compressed, name = "packed", flags = {.Compressed}, compressed_size = 65536),
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
e := m.entries[100]
testing.expect_value(t, e.size, u64(5000)) // unnamed stream only
- testing.expect_value(t, e.allocated, u64(8192 + 4096 + 65536)) // every stream; compressed at its on-disk size
+ testing.expect_value(t, e.allocated, u64(8192 + 4096 + 65536)) // every stream, measured from run lists
testing.expect_value(t, e.parent, u32(64))
testing.expect_value(t, e.parent_sequence, u16(3))
}
@(test)
-test_later_extent_does_not_double_count :: proc(t: ^testing.T) {
+test_fragmented_extents_sum_their_own_clusters :: proc(t: ^testing.T) {
+ // Both extents repeat the header sizes (8192 allocated) but each maps one cluster.
rec := build_record(
{
resident(.File_Name, file_name_value(make_ref(RECORD_ROOT, 5), "frag.bin", .Win32)),
@@ -342,10 +347,38 @@ test_later_extent_does_not_double_count :: proc(t: ^testing.T) {
},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].allocated, u64(8192))
+ testing.expect_value(t, m.entries[100].size, u64(8000))
+}
+
+@(test)
+test_unallocated_runlist_counts_nothing :: proc(t: ^testing.T) {
+ // $BadClus:$Bad spans the whole volume with sparse runs and no sparse flag.
+ rec := build_record(
+ {
+ resident(.File_Name, file_name_value(make_ref(RECORD_ROOT, 5), "$BadClus", .Win32_And_Dos)),
+ nonresident(.Data, 0, 0, []byte{0x00}),
+ nonresident(.Data, 1 << 40, 1 << 40, []byte{0x04, 0x00, 0x00, 0x00, 0x10, 0x00}, name = "$Bad"),
+ },
+ record_number = RECORD_BAD_CLUS,
+ )
+ m: Mft
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
+ defer mft_destroy(&m)
+ add(t, &m, rec)
+ testing.expect_value(t, m.entries[RECORD_BAD_CLUS].allocated, u64(0))
+}
+
+@(test)
+test_runlist_allocated_clusters :: proc(t: ^testing.T) {
+ clusters, ok := runlist_allocated_clusters([]byte{0x11, 0x10, 0x30, 0x01, 0xF0, 0x00})
+ testing.expect(t, ok)
+ testing.expect_value(t, clusters, u64(16))
+ _, ok = runlist_allocated_clusters([]byte{0x21, 0x18})
+ testing.expect(t, !ok)
}
@(test)
@@ -365,7 +398,7 @@ test_extension_record_and_hard_links :: proc(t: ^testing.T) {
base = make_ref(100, 9),
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
// The extension arrives before its base, as it can on disk.
@@ -396,7 +429,7 @@ test_free_record_is_ignored :: proc(t: ^testing.T) {
flags = {},
)
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
add(t, &m, rec)
testing.expect_value(t, m.entries[100].name, "")
@@ -408,7 +441,7 @@ test_free_record_is_ignored :: proc(t: ^testing.T) {
@(test)
test_path :: proc(t: ^testing.T) {
m: Mft
- testing.expect_value(t, mft_init(&m, 128), Error.None)
+ testing.expect_value(t, mft_init(&m, 128, 4096), Error.None)
defer mft_destroy(&m)
root := make_ref(RECORD_ROOT, 5)
diff --git a/ntfs/reader.odin b/ntfs/reader.odin
@@ -109,7 +109,7 @@ read_mft_from_volume :: proc(v: ^Volume, opts := Read_Options{}, allocator := co
}
record_count := int(mft_bytes / u64(record_size))
- if init_err := mft_init(&m, record_count, allocator); init_err != nil {
+ if init_err := mft_init(&m, record_count, cluster, allocator); init_err != nil {
return {}, init_err
}
m.boot = boot
diff --git a/ntfs/record.odin b/ntfs/record.odin
@@ -111,13 +111,27 @@ Attribute :: struct {
runlist: []byte, // raw mapping pairs; decode with decode_runlist
}
-// Bytes this attribute extent actually occupies on disk. Resident values live inside
-// the record and cost nothing extra. Compressed and sparse attributes report their
-// logical allocation in allocated_size, so the real figure is compressed_size.
-attr_disk_size :: proc "contextless" (a: Attribute) -> u64 {
+/*
+Bytes this attribute extent actually occupies on disk.
+
+Resident values live inside the record and cost nothing extra. For non-resident
+extents the run list is authoritative: it says exactly which clusters are allocated,
+which is correct for plain, compressed, and sparse attributes alike, and for the
+system files whose headers claim allocation they do not have. Each extent's run list
+covers only that extent's VCN range, so summing over every extent of a fragmented
+attribute gives the whole. The header sizes are only a fallback for a malformed run
+list, and then only on the first extent, since later extents repeat them.
+*/
+attr_disk_size :: proc "contextless" (a: Attribute, bytes_per_cluster: u64) -> u64 {
if !a.non_resident {
return 0
}
+ if clusters, ok := runlist_allocated_clusters(a.runlist); ok {
+ return clusters * bytes_per_cluster
+ }
+ if a.lowest_vcn != 0 {
+ return 0
+ }
if .Compressed in a.flags || .Sparse in a.flags {
return a.compressed_size
}
diff --git a/ntfs/runlist.odin b/ntfs/runlist.odin
@@ -60,6 +60,35 @@ decode_runlist :: proc(b: []byte, first_vcn: u64, allocator := context.allocator
return out[:], .None
}
+/*
+Count the clusters a run list actually allocates, skipping sparse runs, without
+building the run array. This is the ground truth for disk usage: allocated_size in
+the attribute header is logical for compressed and sparse attributes, and some system
+files ($BadClus:$Bad, which spans the whole volume) have entirely unallocated run
+lists with no sparse flag at all.
+*/
+runlist_allocated_clusters :: proc "contextless" (b: []byte) -> (clusters: u64, ok: bool) {
+ i := 0
+ for i < len(b) {
+ h := b[i]
+ i += 1
+ if h == 0 {
+ return clusters, true
+ }
+ len_size := int(h & 0xF)
+ ofs_size := int(h >> 4)
+ if len_size == 0 || len_size > 8 || ofs_size > 8 || i + len_size + ofs_size > len(b) {
+ return 0, false
+ }
+ length := read_uint_le(b[i:i + len_size])
+ i += len_size + ofs_size
+ if ofs_size != 0 {
+ clusters += length
+ }
+ }
+ return clusters, true
+}
+
// Total clusters covered by a run list, including sparse runs.
runlist_clusters :: proc(runs: []Run) -> u64 {
total: u64
