From 360c69daf7a83ec31c384390d9ec37064b78b40e Mon Sep 17 00:00:00 2001 From: Amlal Date: Sat, 26 Apr 2025 16:56:37 +0200 Subject: dev, kernel, tooling: add HeFS tooling and early allocation stategy. why? - we need to format it externally too. - a in-kernel format doesn't handle every case. - and i have to fix the createdir, createfile methods for correct inode dir linking. Signed-off-by: Amlal --- .gitignore | 2 + compile_flags.txt | 2 + dev/kernel/FSKit/HeFS.h | 7 +- dev/kernel/src/FS/HeFS.cc | 72 ++++++++++---------- tooling/dist/.keep | 0 tooling/hefs.h | 161 ++++++++++++++++++++++++++++++++++++++++++++ tooling/mkfs.hefs.cc | 165 ++++++++++++++++++++++++++++++++++++++++++++++ tooling/mkfs.hefs.json | 16 +++++ 8 files changed, 389 insertions(+), 36 deletions(-) create mode 100644 tooling/dist/.keep create mode 100644 tooling/hefs.h create mode 100644 tooling/mkfs.hefs.cc create mode 100644 tooling/mkfs.hefs.json diff --git a/.gitignore b/.gitignore index df03c3e1..fc656b48 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ public/frameworks/*/dist/* neoskrnl/neoskrnl.xcodeproj/project.xcworkspace/xcshareddata/ +tooling/dist/mkfs.hefs + html/ latex/ diff --git a/compile_flags.txt b/compile_flags.txt index cef28da1..be8085bd 100644 --- a/compile_flags.txt +++ b/compile_flags.txt @@ -13,6 +13,8 @@ -Ipublic/tools/open -Ipublic/frameworks -Idev/boot/BootKit +-Itooling/ +-I./ -std=c++20 -D__NE_AMD64__ -D__NEOSKRNL__ diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index 693ec2d3..a2dd9fe0 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -29,7 +29,10 @@ #define kHeFSDefaultVoluneName u"HeFS Volume" #define kHeFSDIMBootDir u"boot-x/dir" -#define kHeFSDIMBootFile u"boot-x/file" +#define kHeFSMIMEBootFile u"boot-x/file" + +#define kHeFSDIMSystemDir u"system-x/dir" +#define kHeFSMIMESystemFile u"system-x/file" #define kHeFSSearchAllStr u"*" @@ -123,7 +126,7 @@ struct PACKED HEFS_BOOT_NODE final { }; inline constexpr Kernel::ATime kHeFSTimeInvalid = 0x0000000000000000; -inline constexpr Kernel::ATime kHeFSTimeMax = 0xFFFFFFFFFFFFFFFF; +inline constexpr Kernel::ATime kHeFSTimeMax = 0xFFFFFFFFFFFFFFFF - 1; /// @brief HeFS index node. /// @details This structure is used to store the file information of a file. diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 81316ba5..13d79c8f 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -78,29 +78,20 @@ namespace Detail { hefsi_traverse_tree(HEFS_INDEX_NODE_DIRECTORY* dir, HEFS_BOOT_NODE* node, Lba& start) { NE_UNUSED(node); - if (!dir) { - ke_panic(RUNTIME_CHECK_FILESYSTEM, "Error: Invalid directory node in RB-Tree traversal.\r"); + if (!dir || !node) { + ke_panic(RUNTIME_CHECK_FILESYSTEM, "Error: Invalid directory node/boot_node in RB-Tree traversal."); } - start = dir->fNext; - - if (dir->fColor == kHeFSBlack && start == 0) { - if (dir->fParent != 0) start = dir->fParent; - } else if (dir->fColor == kHeFSRed && start == 0) { - if (dir->fChild != 0) - start = dir->fChild; - else if (dir->fNext != 0) - start = dir->fNext; - else if (dir->fPrev != 0) - start = dir->fPrev; - else - start = dir->fParent; - } - - if (start == 0) { - ke_panic(RUNTIME_CHECK_FILESYSTEM, "Error: Invalid start value in RB-Tree traversal.\r"); - - return; + if (dir->fChild != 0) { + start = dir->fChild; + } else if (dir->fNext != 0) { + start = dir->fNext; + } else if (dir->fParent != 0) { + start = dir->fParent; + } else if (dir->fPrev != 0) { + start = dir->fPrev; + } else { + start = node->fStartIND; } } @@ -324,13 +315,16 @@ namespace Detail { return NO; } - if (parent->fDeleted) { + if (parent->fDeleted || + !parent->fCreated) { mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); mnt->fPacket.fPacketContent = dir; mnt->fOutput(mnt->fPacket); + hefsi_balance_filesystem(root, mnt); + return YES; } @@ -866,14 +860,14 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input wrt_copy_memory((VoidPtr) u"/", root_dir->fName, wrt_string_len(u"/")); wrt_copy_memory((VoidPtr) kHeFSDIMBootDir, root_dir->fDim, wrt_string_len(kHeFSDIMBootDir)); - root_dir->fKind = kHeFSFileKindDirectory; - root_dir->fColor = kHeFSBlack; // Every RB-Tree root starts black. (a condition of the algorithm) - root_dir->fParent = 0; // No parent (it's the real root) - root_dir->fChild = 0; // No children yet - root_dir->fNext = 0; // No next - root_dir->fPrev = 0; // No previous + root_dir->fKind = kHeFSFileKindDirectory; + root_dir->fColor = kHeFSBlack; // Every RB-Tree root starts black. (a condition of the algorithm) + root_dir->fParent = root->fStartIND; // No parent (it's the real root) + root_dir->fChild = root->fEndIND; // No children yet + root_dir->fNext = 0; // No next + root_dir->fPrev = 0; // No previous - root_dir->fEntryCount = 0; + root_dir->fEntryCount = 1; drive->fPacket.fPacketLba = drive->fLbaStart + sizeof(HEFS_BOOT_NODE) + sizeof(HEFS_BOOT_NODE); drive->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); @@ -881,6 +875,8 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input drive->fOutput(drive->fPacket); + Detail::hefsi_balance_filesystem(root, drive); + delete root_dir; delete root; @@ -942,6 +938,8 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu drive->fInput(drive->fPacket); + Detail::hefsi_balance_filesystem(root, drive); + auto dirent = Detail::hefs_fetch_index_node_directory(root, drive, dir); if (dirent) { @@ -963,20 +961,22 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu wrt_copy_memory((VoidPtr) dir, dirent->fName, wrt_string_len(dir)); dirent->fAccessed = 0; - dirent->fCreated = 0; + dirent->fCreated = 0; /// TODO: Add the current time. dirent->fDeleted = 0; dirent->fModified = 0; dirent->fEntryCount = 0; - dirent->fParent = 0; // No parent (it's the real root) - dirent->fChild = 0; // No children yet - dirent->fNext = 0; // No next - dirent->fPrev = 0; // No previous + dirent->fParent = root->fStartIND; // No parent (it's the real root) + dirent->fChild = root->fEndIND; // No children yet + dirent->fNext = 0; // No next + dirent->fPrev = 0; // No previous dirent->fKind = kHeFSFileKindDirectory; dirent->fFlags = flags; dirent->fChecksum = 0; + dirent->fEntryCount = 1; + if (Detail::hefs_allocate_index_directory_node(root, drive, dirent)) { delete dirent; dirent = nullptr; @@ -1033,6 +1033,8 @@ _Output Bool HeFileSystemParser::CreateFile(_Input DriveTrait* drive, _Input con drive->fInput(drive->fPacket); + Detail::hefsi_balance_filesystem(root, drive); + auto dirent = Detail::hefs_fetch_index_node_directory(root, drive, dir); if (!dirent) { @@ -1077,6 +1079,8 @@ _Output Bool HeFileSystemParser::CreateFile(_Input DriveTrait* drive, _Input con return NO; } +/// @brief Initialize the HeFS filesystem. +/// @return To check its status, see err_local_get(). Boolean fs_init_hefs(Void) noexcept { kout << "Creating main disk with HeFS in it...\r"; diff --git a/tooling/dist/.keep b/tooling/dist/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tooling/hefs.h b/tooling/hefs.h new file mode 100644 index 00000000..a85cb2bc --- /dev/null +++ b/tooling/hefs.h @@ -0,0 +1,161 @@ +/* ------------------------------------------- + + Copyright (C) 2025, Amlal El Mahrouss, all rights reserved. + +------------------------------------------- */ + +#pragma once + +#include +#include + +#define kHeFSVersion (0x0101) +#define kHeFSMagic " HeFS" +#define kHeFSMagicLen (8) + +#define kHeFSFileNameLen (256U) +#define kHeFSPartNameLen (128U) + +#define kHeFSMinimumDiskSize (gib_cast(4)) + +#define kHeFSDefaultVoluneName u"HeFS Volume" + +#define kHeFSDIMBootDir u"boot-x/dir" +#define kHeFSMIMEBootFile u"boot-x/file" + +#define kHeFSDIMSystemDir u"system-x/dir" +#define kHeFSMIMESystemFile u"system-x/file" + +#define kHeFSSearchAllStr u"*" + +namespace mkfs::hefs { +enum { + kHeFSHardDrive = 0xC0, // Hard Drive + kHeFSSolidStateDrive = 0xC1, // Solid State Drive + kHeFSOpticalDrive = 0x0C, // Blu-Ray/DVD + kHeFSMassStorageDevice = 0xCC, // USB + kHeFSScsiDrive = 0xC4, // SCSI Hard Drive + kHeFSFlashDrive = 0xC6, + kHeFSUnknown = 0xFF, // Unknown device. + kHeFSDriveCount = 7, +}; + +enum { + kHeFSStatusUnlocked = 0x18, + kHeFSStatusLocked, + kHeFSStatusError, + kHeFSStatusInvalid, + kHeFSStatusCount, +}; + +enum { + kHeFSEncodingUTF8 = 0x00, + kHeFSEncodingUTF16, + kHeFSEncodingUTF32, + kHeFSEncodingUTF16BE, + kHeFSEncodingUTF16LE, + kHeFSEncodingUTF32BE, + kHeFSEncodingUTF32LE, + kHeFSEncodingUTF8BE, + kHeFSEncodingUTF8LE, + kHeFSEncodingBinary, + kHeFSEncodingCount, +}; + +// Constants +constexpr std::size_t kHeFSBlockCount = 16; + +// Types +using ATime = std::uint64_t; + +inline constexpr uint16_t kHeFSFileKindRegular = 0x00; +inline constexpr uint16_t kHeFSFileKindDirectory = 0x01; +inline constexpr uint16_t kHeFSFileKindBlock = 0x02; +inline constexpr uint16_t kHeFSFileKindCharacter = 0x03; +inline constexpr uint16_t kHeFSFileKindFIFO = 0x04; +inline constexpr uint16_t kHeFSFileKindSocket = 0x05; +inline constexpr uint16_t kHeFSFileKindSymbolicLink = 0x06; +inline constexpr uint16_t kHeFSFileKindUnknown = 0x07; +inline constexpr uint16_t kHeFSFileKindCount = 0x08; + +// Basic Time Constants +inline constexpr ATime kHeFSTimeInvalid = 0x0000000000000000; +inline constexpr ATime kHeFSTimeMax = 0xFFFFFFFFFFFFFFFF - 1; + +// Boot Node (Superblock Equivalent) +struct alignas(8) BootNode { + char magic[kHeFSMagicLen]{}; + char16_t volumeName[kHeFSPartNameLen]{}; + std::uint32_t version{}; + std::uint64_t badSectors{}; + std::uint64_t sectorCount{}; + std::uint64_t sectorSize{}; + std::uint32_t checksum{}; + std::uint8_t diskKind{}; + std::uint8_t encoding{}; + std::uint64_t startIND{}; + std::uint64_t endIND{}; + std::uint64_t indCount{}; + std::uint64_t diskSize{}; + std::uint16_t diskStatus{}; + std::uint16_t diskFlags{}; + std::uint16_t vid{}; + std::uint64_t reserved{}; + std::uint64_t reserved2{}; + std::uint64_t reserved3{}; + std::uint64_t reserved4{}; +}; + +// File Node (Index Node) +struct alignas(8) IndexNode { + char16_t name[kHeFSFileNameLen]{}; + std::uint32_t flags{}; + std::uint16_t kind{}; + std::uint32_t size{}; + std::uint32_t checksum{}; + std::uint32_t recoverChecksum{}; + std::uint32_t blockChecksum{}; + std::uint32_t linkChecksum{}; + char16_t mime[kHeFSFileNameLen]{}; + bool symbolicLink{false}; + ATime created{}; + ATime accessed{}; + ATime modified{}; + ATime deleted{}; + std::uint32_t uid{}; + std::uint32_t gid{}; + std::uint32_t mode{}; + std::uint64_t blockLinkStart[kHeFSBlockCount]{}; + std::uint64_t blockLinkEnd[kHeFSBlockCount]{}; + std::uint64_t blockStart[kHeFSBlockCount]{}; + std::uint64_t blockEnd[kHeFSBlockCount]{}; + std::uint64_t blockRecoveryStart[kHeFSBlockCount]{}; + std::uint64_t blockRecoveryEnd[kHeFSBlockCount]{}; +}; + +// Directory Node (Red-Black Tree Node) +struct alignas(8) IndexNodeDirectory { + char16_t name[kHeFSFileNameLen]{}; + std::uint32_t flags{}; + std::uint16_t kind{}; + std::uint32_t entryCount{}; + std::uint32_t checksum{}; + std::uint32_t indexNodeChecksum{}; + char16_t dim[kHeFSFileNameLen]{}; + ATime created{}; + ATime accessed{}; + ATime modified{}; + ATime deleted{}; + std::uint32_t uid{}; + std::uint32_t gid{}; + std::uint32_t mode{}; + std::uint64_t indexNodeStart[kHeFSBlockCount]{}; + std::uint64_t indexNodeEnd[kHeFSBlockCount]{}; + std::uint8_t color{}; + std::uint64_t next{}; + std::uint64_t prev{}; + std::uint64_t child{}; + std::uint64_t parent{}; +}; + +} // namespace mkfs::hefs diff --git a/tooling/mkfs.hefs.cc b/tooling/mkfs.hefs.cc new file mode 100644 index 00000000..15b18a8a --- /dev/null +++ b/tooling/mkfs.hefs.cc @@ -0,0 +1,165 @@ +/* ------------------------------------------- + + Copyright (C) 2025, Amlal El Mahrouss, all rights reserved. + +------------------------------------------- */ + +#include +#include +#include +#include +#include + +/// @internal +namespace detail { +/// @brief Helper function to get the option value from command line arguments. +template +static std::basic_string get_option(const std::basic_string& args, + const std::basic_string& option) { + size_t pos = args.find(option + CharType('=')); + + if (pos != std::string::npos) { + size_t start = pos + option.length() + 1; + size_t end = args.find(' ', start); + return args.substr(start, end - start); + } + + return std::basic_string{}; +} +} // namespace detail + +static size_t kDiskSize = 1024 * 1024 * 1024 * 4UL; +static uint16_t kVersion = kHeFSVersion; +static std::u16string kLabel = kHeFSDefaultVoluneName; +static size_t kSectorSize = 512; + +int main(int argc, char** argv) { + if (argc < 2) { + std::cerr << "mkfs.hefs: Error: Missing required arguments." << std::endl; + std::cerr << "mkfs.hefs: Usage: mkfs.hefs -L