From 00e314410e6049c2ddbcb4861c04af6b06eeeea3 Mon Sep 17 00:00:00 2001 From: Amlal El Mahrouss Date: Mon, 21 Apr 2025 08:47:04 +0200 Subject: dev, kernel, tools, tooling, tex: add mk_app tool, see details. - Patch HeFS implementation file, working on a allocation function now. - Generated LaTeX specs from source code. - Add mk.{hefs, nefs} tools for future formatting (with diutil first) - Add python tool to generate user apps for NeKernel. Signed-off-by: Amlal El Mahrouss --- docs/tex/hefs.tex | 121 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 15 deletions(-) (limited to 'docs') diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index e74cf5e3..3370af48 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -1,32 +1,123 @@ \documentclass{article} -\usepackage[a4paper,margin=1in]{geometry} +\usepackage[a4paper, margin=1in]{geometry} +\usepackage{amsmath, amssymb} \usepackage{listings} \usepackage{xcolor} -\usepackage{amsmath} -\usepackage{hyperref} -\usepackage{longtable} -\usepackage{titlesec} -\usepackage{fancyhdr} -\usepackage{caption} \usepackage{graphicx} +\usepackage{enumitem} +\usepackage{caption} +\usepackage{longtable} -\definecolor{codegray}{gray}{0.95} -\lstset{ - backgroundcolor=\color{codegray}, +\definecolor{lightgray}{gray}{0.95} + +\lstdefinestyle{cstyle}{ + language=C++, + backgroundcolor=\color{lightgray}, basicstyle=\ttfamily\small, + keywordstyle=\color{blue}, + commentstyle=\color{green!60!black}, + stringstyle=\color{orange}, + numbers=left, + numberstyle=\tiny, breaklines=true, frame=single, - tabsize=4, - language=C++, showstringspaces=false } -\title{HeFS: High-Throughput Extended File System Specification} +\title{HeFS (Hierarchical Embedded File System) Specification} \author{Amlal El Mahrouss} -\date{2025} +\date{2024–2025} \begin{document} \maketitle -\end{document} +\section{Overview} +HeFS is a custom filesystem developed as part of the NeKernel project. It offers a journaling-like inode tree structure using red-black tree-inspired navigation for directories. Designed for robust use in desktop and server workloads, it supports various encoding modes and drive types. + +\section{Boot Node Structure} +\begin{longtable}{|l|l|p{8cm}|} +\hline +\textbf{Field} & \textbf{Type} & \textbf{Description} \\ +\hline +\verb|fMagic| & \verb|char[8]| & Filesystem magic (" HeFS") \\ +\verb|fVolName| & \verb|Utf16Char[128]| & Volume name \\ +\verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0100) \\ +\verb|fSectorCount| & \verb|UInt64| & Total sector count \\ +\verb|fSectorSize| & \verb|UInt64| & Size of each sector \\ +\verb|fDriveKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\ +\verb|fEncoding| & \verb|UInt8| & Encoding mode (UTF-8, UTF-16, etc.) \\ +\verb|fStartIND| & \verb|UInt64| & Starting LBA of inode tree \\ +\verb|fEndIND| & \verb|UInt64| & Ending LBA of inode tree \\ +\verb|fChecksum| & \verb|UInt32| & CRC32 of boot node \\ +\verb|fDiskSize| & \verb|UInt64| & Logical size of the disk \\ +\hline +\end{longtable} + +\section{File Types and Flags} +\subsection*{File Kinds} +\begin{itemize}[label=--] + \item \verb|0x00| — Regular File + \item \verb|0x01| — Directory + \item \verb|0x02| — Block Device + \item \verb|0x03| — Character Device + \item \verb|0x04| — FIFO + \item \verb|0x05| — Socket + \item \verb|0x06| — Symbolic Link +\end{itemize} + +\subsection*{Drive Types} +\begin{itemize}[label=--] + \item \verb|0xC0| — Hard Drive + \item \verb|0xC1| — SSD + \item \verb|0xCC| — Mass Storage Device (USB) +\end{itemize} + +\section{Index Node Structure} +The inode tree allows for fast access and recovery via dual start/end block mappings and timestamped metadata. + +\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE structure}] +struct HEFS_INDEX_NODE { + Utf16Char fName[256]; + UInt32 fFlags; + UInt16 fKind; + UInt32 fSize; + UInt32 fChecksum, fRecoverChecksum, fBlockChecksum, fLinkChecksum; + ATime fCreated, fAccessed, fModified, fDeleted; + UInt32 fUID, fGID; + UInt32 fMode; + UInt64 fBlockLinkStart[16], fBlockLinkEnd[16]; + UInt64 fBlockStart[16], fBlockEnd[16]; + UInt64 fBlockRecoveryStart[16], fBlockRecoveryEnd[16]; +}; +\end{lstlisting} + +\section{Directory Node Structure} +Each directory is a red-black tree node with child and sibling pointers. + +\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE\_DIRECTORY structure}] +struct HEFS_INDEX_NODE_DIRECTORY { + Utf16Char fName[256]; + UInt32 fFlags; + UInt16 fKind; + UInt32 fSize; + UInt32 fChecksum, fIndexNodeChecksum; + ATime fCreated, fAccessed, fModified, fDeleted; + UInt32 fUID, fGID; + UInt32 fMode; + UInt64 fIndexNodeStart[16], fIndexNodeEnd[16]; + UInt8 fColor; + Lba fNext, fPrev, fChild, fParent; +}; +\end{lstlisting} + +\section{Future Work} +Planned extensions include: +\begin{itemize} + \item Journaling recovery logic + \item Advanced permission flags + \item Logical volume management support + \item Cross-filesystem linking (EPM integration) +\end{itemize} + +\end{document} \ No newline at end of file -- cgit v1.2.3 From f8a49e346e45daaf1463b658b386040f8843fa61 Mon Sep 17 00:00:00 2001 From: Amlal El Mahrouss Date: Thu, 24 Apr 2025 09:14:06 +0200 Subject: dev, kernel, boot: HeFS, Scheduler, BootZ: see below. Refactored HeFS node handling and API for extensibility, added directory metadata, improved wildcard querying, and updated documentation accordingly. Signed-off-by: Amlal El Mahrouss --- dev/boot/amd64-desktop.make | 2 +- dev/boot/src/HEL/AMD64/BootEFI.cc | 6 +- dev/kernel/FSKit/HeFS.h | 11 +- dev/kernel/HALKit/AMD64/HalKernelMain.cc | 2 + dev/kernel/src/FS/HeFS.cc | 180 ++++++++++++++++++++++++++++--- docs/tex/hefs.tex | 30 +++++- 6 files changed, 202 insertions(+), 29 deletions(-) (limited to 'docs') diff --git a/dev/boot/amd64-desktop.make b/dev/boot/amd64-desktop.make index 00b37b94..652ba31e 100644 --- a/dev/boot/amd64-desktop.make +++ b/dev/boot/amd64-desktop.make @@ -97,7 +97,7 @@ all: compile-amd64 .PHONY: disk disk: dd if=/dev/zero of=$(BOOT) bs=7M count=100 - mformat -i $(BOOT) -F -v "NEKERNEL-ESP" + mformat -i $(BOOT) -F -v "ESP" ifneq ($(DEBUG_SUPPORT), ) diff --git a/dev/boot/src/HEL/AMD64/BootEFI.cc b/dev/boot/src/HEL/AMD64/BootEFI.cc index 0fc21860..6411383d 100644 --- a/dev/boot/src/HEL/AMD64/BootEFI.cc +++ b/dev/boot/src/HEL/AMD64/BootEFI.cc @@ -190,7 +190,7 @@ EFI_EXTERN_C EFI_API Int32 BootloaderMain(EfiHandlePtr image_handle, if (reader_syschk.Blob()) { syschk_thread = new Boot::BootThread(reader_syschk.Blob()); - syschk_thread->SetName("BootZ\\System Check"); + syschk_thread->SetName("BootZ/SysChk"); syschk_thread->Start(handover_hdr, NO); } @@ -258,7 +258,7 @@ EFI_EXTERN_C EFI_API Int32 BootloaderMain(EfiHandlePtr image_handle, auto kernel_thread = Boot::BootThread(reader_kernel.Blob()); - kernel_thread.SetName("BootZ\\NeKernel"); + kernel_thread.SetName("BootZ/NeKernel"); handover_hdr->f_KernelImage = reader_kernel.Blob(); handover_hdr->f_KernelSz = reader_kernel.Size(); @@ -273,7 +273,7 @@ EFI_EXTERN_C EFI_API Int32 BootloaderMain(EfiHandlePtr image_handle, return kEfiFail; auto netboot_thread = Boot::BootThread(reader_netboot.Blob()); - netboot_thread.SetName("BootZ\\BootNet"); + netboot_thread.SetName("BootZ/BootNet"); return netboot_thread.Start(handover_hdr, NO); } diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index 8af6c568..0ad47739 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -26,7 +26,9 @@ #define kHeFSMinimumDiskSize (gib_cast(4)) -#define kHeFSDefaultVoluneName "HeFS Volume" +#define kHeFSDefaultVoluneName u"HeFS Volume" + +#define kHeFSSearchAllStr u"*" struct HEFS_BOOT_NODE; struct HEFS_INDEX_NODE; @@ -162,7 +164,7 @@ struct PACKED ALIGN(8) HEFS_INDEX_NODE_DIRECTORY final Kernel::UInt32 fFlags; /// @brief File flags. Kernel::UInt16 fKind; /// @brief File kind. (Regular, Directory, Block, Character, FIFO, Socket, Symbolic Link, Unknown). - Kernel::UInt32 fSize; /// @brief Size of the directory. + Kernel::UInt32 fEntryCount; /// @brief Entry Count of this directory inode. Kernel::UInt32 fChecksum, fIndexNodeChecksum; /// @brief Checksum of the file, index node checksum. Kernel::ATime fCreated, fAccessed, fModified, fDeleted; /// @brief File timestamps. @@ -343,11 +345,6 @@ namespace Kernel::Detail return "Unknown"; } } - - inline SizeT hefs_get_block_size(SizeT block_size) noexcept - { - return block_size * kHeFSBlockCount; - } } // namespace Kernel::Detail namespace Kernel diff --git a/dev/kernel/HALKit/AMD64/HalKernelMain.cc b/dev/kernel/HALKit/AMD64/HalKernelMain.cc index 3686a48f..d59e9e86 100644 --- a/dev/kernel/HALKit/AMD64/HalKernelMain.cc +++ b/dev/kernel/HALKit/AMD64/HalKernelMain.cc @@ -98,6 +98,8 @@ EXTERN_C Kernel::Void hal_real_init(Kernel::Void) noexcept idt_loader.Load(idt_reg); + /// after the scheduler runs, we must look over teams, every 200ms in order to schedule every process according to their affinity fairly. + auto constexpr kSchedTeamSwitchMS = 200U; /// @brief Team switch time in milliseconds. Kernel::HardwareTimer timer(rtl_ms(kSchedTeamSwitchMS)); diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 87decf6a..cfa8baae 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -4,6 +4,7 @@ ------------------------------------------- */ +#include "NewKit/Defines.h" #ifdef __FSKIT_INCLUDES_HEFS__ #include @@ -38,7 +39,15 @@ namespace Kernel /// @param dir_name The name of the directory. /// @param file_name The name of the file. /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown). - STATIC ATTRIBUTE(unused) _Output HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept; + STATIC ATTRIBUTE(unused) _Output HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind, SizeT* cnt) noexcept; + + /// @brief Get the index node size. + /// @param root The root node of the filesystem. + /// @param mnt The drive to read from. + /// @param dir_name The name of the directory. + /// @param file_name The name of the file. + /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown). + STATIC ATTRIBUTE(unused) _Output SizeT hefs_fetch_index_node_size(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept; /// @brief Allocate a new index node. /// @param root The root node of the filesystem. @@ -186,16 +195,125 @@ namespace Kernel mnt->fOutput(mnt->fPacket); } + /// @brief Get the index node size. + /// @param root The root node of the filesystem. + /// @param mnt The drive to read from. + /// @param dir_name The name of the directory. + /// @param file_name The name of the file. + /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown). + STATIC ATTRIBUTE(unused) _Output SizeT hefs_fetch_index_node_size(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept + { + if (root && mnt) + { + HEFS_INDEX_NODE* node = new HEFS_INDEX_NODE(); + HEFS_INDEX_NODE_DIRECTORY* dir = new HEFS_INDEX_NODE_DIRECTORY(); + + SizeT sz = 0UL; + + auto start = root->fStartIND; + auto end = root->fEndIND; + + auto hop_watch = 0; + + while (YES) + { + if (start > end) + { + kout << "Error: Invalid start/end values.\r"; + break; + } + + if (hop_watch > 100) + { + kout << "Error: Hop watch exceeded, filesystem is stalling.\r"; + break; + } + + if (start == 0) + { + ++hop_watch; + start = root->fStartIND; + } + + mnt->fPacket.fPacketLba = start; + mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); + mnt->fPacket.fPacketContent = dir; + + mnt->fInput(mnt->fPacket); + + if (!mnt->fPacket.fPacketGood) + { + delete node; + delete dir; + + dir = nullptr; + node = nullptr; + + err_global_get() = kErrorFileNotFound; + + return 0; + } + + if (dir->fKind == kHeFSFileKindDirectory) + { + if (KStringBuilder::Equals(dir_name, dir->fName) || + KStringBuilder::Equals(dir_name, kHeFSSearchAllStr)) + { + for (SizeT inode_index = 0UL; inode_index < kHeFSBlockCount; inode_index += 2) + { + if (dir->fIndexNodeStart[inode_index]) + { + mnt->fPacket.fPacketLba = dir->fIndexNodeStart[inode_index]; + mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); + mnt->fPacket.fPacketContent = node; + mnt->fInput(mnt->fPacket); + } + else if (dir->fIndexNodeEnd[inode_index]) + { + mnt->fPacket.fPacketLba = dir->fIndexNodeEnd[inode_index]; + mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); + mnt->fPacket.fPacketContent = node; + mnt->fInput(mnt->fPacket); + } + + if (KStringBuilder::Equals(file_name, node->fName) && node->fKind == kind) + { + if (node->fKind == kHeFSFileKindDirectory) + { + sz += hefs_fetch_index_node_size(root, mnt, dir_name, file_name, kind); + } + else + { + sz = node->fSize; + } + + return sz; + } + } + } + } + } + + err_global_get() = kErrorSuccess; + return sz; + } + + err_global_get() = kErrorFileNotFound; + return 0; + } + /// @brief Get the index node of a file or directory. /// @param root The root node of the filesystem. /// @param mnt The drive to read from. /// @param dir_name The name of the directory. /// @param file_name The name of the file. /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown). - STATIC ATTRIBUTE(unused) _Output HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept + STATIC ATTRIBUTE(unused) _Output HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind, SizeT* cnt) noexcept { if (root && mnt) { + HEFS_INDEX_NODE* node_arr = new HEFS_INDEX_NODE[*cnt]; + HEFS_INDEX_NODE* node = new HEFS_INDEX_NODE(); HEFS_INDEX_NODE_DIRECTORY* dir = new HEFS_INDEX_NODE_DIRECTORY(); @@ -208,6 +326,8 @@ namespace Kernel auto start = root->fStartIND; auto end = root->fEndIND; + auto start_cnt = 0UL; + auto hop_watch = 0; while (YES) @@ -251,7 +371,8 @@ namespace Kernel if (dir->fKind == kHeFSFileKindDirectory) { - if (KStringBuilder::Equals(dir_name, dir->fName)) + if (KStringBuilder::Equals(dir_name, dir->fName) || + KStringBuilder::Equals(dir_name, kHeFSSearchAllStr)) { for (SizeT inode_index = 0UL; inode_index < kHeFSBlockCount; inode_index += 2) { @@ -271,7 +392,14 @@ namespace Kernel delete dir; dir = nullptr; - return node; + node_arr[start_cnt] = *node; + ++start_cnt; + + if (start_cnt > *cnt) + { + delete node; + return node_arr; + } } } else @@ -280,9 +408,11 @@ namespace Kernel delete dir; delete node; + delete[] node_arr; - dir = nullptr; - node = nullptr; + dir = nullptr; + node = nullptr; + node_arr = nullptr; return nullptr; } @@ -296,9 +426,11 @@ namespace Kernel delete dir; delete node; + delete[] node_arr; - dir = nullptr; - node = nullptr; + dir = nullptr; + node = nullptr; + node_arr = nullptr; } kout << "Error: Failed to find index node.\r"; @@ -350,18 +482,32 @@ namespace Kernel if (dir->fIndexNodeStart[inode_index] != 0 || dir->fIndexNodeEnd[inode_index] != 0) { - mnt->fPacket.fPacketLba = (!dir->fIndexNodeStart[inode_index]) ? dir->fIndexNodeEnd[inode_index] : dir->fIndexNodeStart[inode_index]; + HEFS_INDEX_NODE prev_node; + + auto lba = (!dir->fIndexNodeStart[inode_index]) ? dir->fIndexNodeEnd[inode_index] : dir->fIndexNodeStart[inode_index]; + + mnt->fPacket.fPacketLba = lba; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); - mnt->fPacket.fPacketContent = node; + mnt->fPacket.fPacketContent = &prev_node; - mnt->fOutput(mnt->fPacket); + mnt->fInput(mnt->fPacket); - if (mnt->fPacket.fPacketGood) + if (prev_node.fDeleted > 0 && + !prev_node.fAccessed) { - delete dir; - dir = nullptr; + mnt->fPacket.fPacketLba = lba; + mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); + mnt->fPacket.fPacketContent = node; + + mnt->fOutput(mnt->fPacket); - return YES; + if (mnt->fPacket.fPacketGood) + { + delete dir; + dir = nullptr; + + return YES; + } } } } @@ -548,7 +694,7 @@ namespace Kernel NE_UNUSED(end_lba); NE_UNUSED(flags); NE_UNUSED(part_name); - + return NO; } @@ -561,7 +707,7 @@ namespace Kernel NE_UNUSED(end_lba); NE_UNUSED(flags); NE_UNUSED(part_name); - + return NO; } } // namespace Kernel diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index 3370af48..d20dfa5e 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -111,13 +111,41 @@ struct HEFS_INDEX_NODE_DIRECTORY { }; \end{lstlisting} +\section{Filesystem Design} + +HeFS is designed to be robust and efficient, with a focus on: +\begin{itemize} + \item Fast access to files and directories + \item Efficient use of disk space + \item Support for large files and directories + \item Compatibility with various drive types + \item Easy recovery from errors + \item Support for advanced features like journaling and permissions +\end{itemize} + +The filesystem uses a red-black tree structure for directories, allowing for efficient insertion, deletion, and searching of files. +Each file and directory is represented by an index node directory, which contains metadata such as timestamps, permissions, and checksums. + +The index node directory itself contains index nodes, which are the actual data structures that store file and directory information. +The filesystem also includes a boot node, which contains information about the filesystem itself, such as the magic number, version, and size. + +The filesystem is designed to be extensible, allowing for future enhancements and features to be added without breaking compatibility with existing data. + +\section{Minimum Requirements} + +HeFS requires at least 4GB of free space on the drive to function properly. +It is recommended to have more than 8GB of free space for optimal performance. +The filesystem is designed to work with various drive types, including hard drives, SSDs, and USB mass storage devices. + +But that really depends on what you do with the filesystem. + \section{Future Work} Planned extensions include: \begin{itemize} \item Journaling recovery logic \item Advanced permission flags \item Logical volume management support - \item Cross-filesystem linking (EPM integration) + \item Cross-filesystem linking (EPM, and GPT integration) \end{itemize} \end{document} \ No newline at end of file -- cgit v1.2.3 From a2bc9eee9184a81d80ab54a09bd239ed26d5293e Mon Sep 17 00:00:00 2001 From: Amlal Date: Fri, 25 Apr 2025 15:24:52 +0200 Subject: dev, kernel: improve nefs format method, hefs format method. Signed-off-by: Amlal --- dev/boot/BootKit/BootKit.h | 39 +------- dev/kernel/FSKit/HeFS.h | 18 ++-- dev/kernel/FSKit/NeFS.h | 9 +- dev/kernel/HALKit/AMD64/HalKernelMain.cc | 2 +- dev/kernel/KernelKit/DriveMgr.h | 2 +- dev/kernel/NewKit/Utils.h | 4 +- dev/kernel/src/FS/HeFS.cc | 98 ++++++++++++++---- dev/kernel/src/FS/NeFS.cc | 167 ++++++++----------------------- dev/kernel/src/WideUtils.cc | 18 +++- docs/tex/hefs.tex | 2 +- modules_pio_x64.sh | 2 +- 11 files changed, 158 insertions(+), 203 deletions(-) (limited to 'docs') diff --git a/dev/boot/BootKit/BootKit.h b/dev/boot/BootKit/BootKit.h index 76821b20..2bf95696 100644 --- a/dev/boot/BootKit/BootKit.h +++ b/dev/boot/BootKit/BootKit.h @@ -275,8 +275,6 @@ class BDiskFormatFactory final { /// @brief Format disk with a specific partition scheme. /// @param part_name partition Name -/// @param blob blos. -/// @param blob_sz n blobs (n * sizeof(blob_struct)). /// @retval True disk has been formatted. /// @retval False failed to format. template @@ -292,42 +290,17 @@ inline Boolean BDiskFormatFactory::Format(const Char* part_name) { return false; } - NEFS_ROOT_PARTITION_BLOCK part{}; - - CopyMem(part.Ident, kNeFSIdent, kNeFSIdentLen - 1); - CopyMem(part.PartitionName, part_name, StrLen(part_name)); - - part.Version = kNeFSVersionInteger; - part.CatalogCount = blob_sz; - part.Kind = BootDev::kSectorSize; - part.SectorSize = kATASectorSize; - part.FreeCatalog = fDiskDev.GetSectorsCount() / sizeof(NEFS_CATALOG_STRUCT); - part.SectorCount = fDiskDev.GetSectorsCount(); - part.FreeSectors = fDiskDev.GetSectorsCount(); - part.StartCatalog = kNeFSCatalogStartAddress; - part.DiskSize = fDiskDev.GetDiskSize(); - part.Flags = kNeFSPartitionTypeBoot | kNeFSPartitionTypeStandard; - - BootTextWriter writer; - - writer << "BootZ: Partition name: " << part.PartitionName << "\r"; - writer << "BootZ: Start: " << part.StartCatalog << "\r"; - writer << "BootZ: Number of catalogs: " << part.CatalogCount << "\r"; - writer << "BootZ: Free catalog: " << part.FreeCatalog << "\r"; - writer << "BootZ: Free sectors: " << part.FreeSectors << "\r"; - writer << "BootZ: Sector size: " << part.SectorSize << "\r"; - EPM_PART_BLOCK epm_boot{}; - const auto kFsName = "NeFS"; + const auto kFsName = "HeFS"; const auto kBlockName = "OS (EPM)"; epm_boot.FsVersion = kNeFSVersionInteger; epm_boot.LbaStart = kNeFSRootCatalogStartAddress; epm_boot.LbaEnd = fDiskDev.GetDiskSize(); - epm_boot.SectorSz = part.SectorSize; + epm_boot.SectorSz = BootDev::kSectorSize; epm_boot.Kind = kEPMNeKernel; - epm_boot.NumBlocks = part.CatalogCount; + epm_boot.NumBlocks = 1; epm_boot.Guid = kEPMNilGuid; @@ -342,11 +315,7 @@ inline Boolean BDiskFormatFactory::Format(const Char* part_name) { fDiskDev.Write((Char*) &epm_boot, sizeof(EPM_PART_BLOCK)); - fDiskDev.Leak().mBase = kNeFSRootCatalogStartAddress; - fDiskDev.Leak().mSize = sizeof(NEFS_ROOT_PARTITION_BLOCK); - - fDiskDev.Write((Char*) &part, sizeof(NEFS_ROOT_PARTITION_BLOCK)); - + BootTextWriter writer; writer.Write(L"BootZ: Drive is EPM formatted.\r"); #elif defined(BOOTZ_VEPM_SUPPORT) NE_UNUSED(part_name); diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index e4da9a47..63725f04 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -100,7 +100,7 @@ struct PACKED HEFS_BOOT_NODE final { Kernel::UInt64 fSectorCount; /// @brief Number of sectors in the filesystem. Kernel::UInt64 fSectorSize; /// @brief Size of the sector. Kernel::UInt32 fChecksum; /// @brief Checksum of the boot node. - Kernel::UInt8 fDriveKind; /// @brief Kind of the drive. (Hard Drive, Solid State Drive, Optical + Kernel::UInt8 fDiskKind; /// @brief Kind of the drive. (Hard Drive, Solid State Drive, Optical /// Drive, etc). Kernel::UInt8 fEncoding; /// @brief Encoding of the filesystem. (UTF-8, UTF-16, etc). Kernel::UInt64 fStartIND; /// @brief Start of the INode tree. @@ -332,7 +332,7 @@ inline const Char* hefs_file_flags_to_string(UInt32 flags) noexcept { } } // namespace Kernel::Detail -namespace Kernel { +namespace Kernel::HeFS { /// @brief HeFS filesystem parser class. /// @details This class is used to parse the HeFS filesystem. class HeFileSystemParser final { @@ -351,17 +351,13 @@ class HeFileSystemParser final { /// @brief Make a EPM+HeFS drive out of the disk. /// @param drive The drive to write on. /// @return If it was sucessful, see err_local_get(). - _Output Bool FormatEPM(_Input _Output DriveTrait* drive, _Input const Lba end_lba, - _Input const Int32 flags, const Char* part_name); - - /// @brief Make a GPT+HeFS drive out of the disk. - /// @param drive The drive to write on. - /// @return If it was sucessful, see err_local_get(). - _Output Bool FormatGPT(_Input _Output DriveTrait* drive, _Input const Lba end_lba, - _Input const Int32 flags, const Char* part_name); + _Output Bool Format(_Input _Output DriveTrait* drive, _Input const Int32 flags, + const Utf16Char* part_name); public: UInt32 mDriveIndex{MountpointInterface::kDriveIndexA}; /// @brief The drive index which this /// filesystem is mounted on. }; -} // namespace Kernel \ No newline at end of file + +Boolean fs_init_hefs(Void) noexcept; +} // namespace Kernel::HeFS \ No newline at end of file diff --git a/dev/kernel/FSKit/NeFS.h b/dev/kernel/FSKit/NeFS.h index 425ab051..2235ca4f 100644 --- a/dev/kernel/FSKit/NeFS.h +++ b/dev/kernel/FSKit/NeFS.h @@ -31,6 +31,8 @@ default. #define kNeFSInvalidCatalog (-1) #define kNeFSCatalogNameLen (256) +#define kNeFSVolumeName "NeFS Volume" + #define kNeFSMinimumDiskSize (mib_cast(8)) #define kNeFSSectorSz (512) @@ -305,11 +307,8 @@ class NeFileSystemParser final { /// @brief Make a EPM+NeFS drive out of the disk. /// @param drive The drive to write on. /// @return If it was sucessful, see err_local_get(). - _Output Bool FormatEPM(_Input _Output DriveTrait* drive, _Input const Lba end_lba, - _Input const Int32 flags, const Char* part_name); - - _Output Bool FormatGPT(_Input _Output DriveTrait* drive, _Input const Lba end_lba, - _Input const Int32 flags, const Char* part_name); + _Output Bool Format(_Input _Output DriveTrait* drive, _Input const Int32 flags, + const Char* part_name); public: UInt32 mDriveIndex{kNeFSSubDriveA}; diff --git a/dev/kernel/HALKit/AMD64/HalKernelMain.cc b/dev/kernel/HALKit/AMD64/HalKernelMain.cc index 82020520..8f4b55cb 100644 --- a/dev/kernel/HALKit/AMD64/HalKernelMain.cc +++ b/dev/kernel/HALKit/AMD64/HalKernelMain.cc @@ -105,7 +105,7 @@ EXTERN_C Int32 hal_init_platform(Kernel::HEL::BootInfoHeader* handover_hdr) { EXTERN_C Kernel::Void hal_real_init(Kernel::Void) noexcept { hal_pre_init_scheduler(); - Kernel::NeFS::fs_init_nefs(); + Kernel::HeFS::fs_init_hefs(); Kernel::HAL::mp_init_cores(kHandoverHeader->f_HardwareTables.f_VendorPtr); diff --git a/dev/kernel/KernelKit/DriveMgr.h b/dev/kernel/KernelKit/DriveMgr.h index e0bed8f5..263b728e 100644 --- a/dev/kernel/KernelKit/DriveMgr.h +++ b/dev/kernel/KernelKit/DriveMgr.h @@ -62,7 +62,7 @@ struct DriveTrait final { } fPacket; Lba fLbaStart{0}, fLbaEnd{0}; - SizeT fSectorSz{512}; + SizeT fSectorSz{kDriveSectorSz}; Void (*fInput)(DrivePacket packet); Void (*fOutput)(DrivePacket packet); diff --git a/dev/kernel/NewKit/Utils.h b/dev/kernel/NewKit/Utils.h index ca1c7d1c..2f0fc60b 100644 --- a/dev/kernel/NewKit/Utils.h +++ b/dev/kernel/NewKit/Utils.h @@ -17,7 +17,6 @@ void rt_zero_memory(voidPtr pointer, Size len); Int rt_string_cmp(const Char* src, const Char* cmp, Size len); const Char* rt_alloc_string(const Char* text); Size rt_string_len(const Char* str); -Size wrt_string_len(const Utf16Char* str); Size rt_string_len(const Char* str, SizeT _len); Boolean rt_to_string(Char* str_out, UInt64 base, Int32 limit); Boolean rt_is_newln(Char chr); @@ -27,4 +26,7 @@ Int rt_to_uppercase(Int c); Int rt_to_lower(Int c); voidPtr rt_string_in_string(const Char* in, const Char* needle); char* rt_string_has_char(Char* str, Char chr); + +Int wrt_copy_memory(const voidPtr src, voidPtr dst, Size len); +Size wrt_string_len(const Utf16Char* str); } // namespace Kernel diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 88837273..5beb8b3f 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -4,6 +4,7 @@ ------------------------------------------- */ +#include "NewKit/Defines.h" #ifdef __FSKIT_INCLUDES_HEFS__ #include @@ -636,34 +637,95 @@ namespace Detail { /// real-time. /// @note This is certainly take longer to format a disk with it, but worth-it in the long run. -namespace Kernel { +namespace Kernel::HeFS { /// @brief Make a EPM+HeFS drive out of the disk. /// @param drive The drive to write on. /// @return If it was sucessful, see err_local_get(). -_Output Bool HeFileSystemParser::FormatEPM(_Input _Output DriveTrait* drive, - _Input const Lba end_lba, _Input const Int32 flags, - const Char* part_name) { +_Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input const Int32 flags, + const Utf16Char* part_name) { NE_UNUSED(drive); - NE_UNUSED(end_lba); NE_UNUSED(flags); NE_UNUSED(part_name); - return NO; + // verify disk. + drive->fVerify(drive->fPacket); + + // if disk isn't good, then error out. + if (false == drive->fPacket.fPacketGood) { + err_global_get() = kErrorDiskIsCorrupted; + return false; + } + + + HEFS_BOOT_NODE* root = new HEFS_BOOT_NODE(); + + if (!root) { + kout << "Error: Failed to allocate memory for boot node.\r"; + return NO; + } + + rt_set_memory(root, 0, sizeof(HEFS_BOOT_NODE)); + + rt_copy_memory((VoidPtr) "fs/hefs-packet", drive->fPacket.fPacketMime, + rt_string_len("fs/hefs-packet")); + + wrt_copy_memory((VoidPtr) part_name, root->fVolName, wrt_string_len(part_name)); + rt_copy_memory((VoidPtr) kHeFSMagic, root->fMagic, sizeof(kHeFSMagic)); + + root->fBadSectors = 0; + + root->fSectorCount = drv_get_sector_count(); + + root->fSectorSize = drive->fSectorSz; + + root->fStartIND = drive->fLbaStart + sizeof(HEFS_BOOT_NODE); + root->fEndIND = drive->fLbaEnd; + + root->fINDCount = root->fEndIND - root->fStartIND; + + root->fDiskSize = drv_get_size(); + root->fDiskStatus = kHeFSStatusUnlocked; + + root->fDiskFlags = flags; + + if (drive->fKind & kMassStorageDrive) { + } else if (drive->fKind & kHeFSOpticalDrive) { + root->fDiskKind = kHeFSOpticalDrive; + } else { + root->fDiskKind = kHeFSUnknown; + } + + root->fReserved = 0; + root->fReserved2 = 0; + root->fReserved3 = 0; + root->fReserved4 = 0; + + root->fChecksum = 0; + + root->fVID = kHeFSInvalidVID; + + drive->fPacket.fPacketLba = drive->fLbaStart; + drive->fPacket.fPacketSize = sizeof(HEFS_BOOT_NODE); + drive->fPacket.fPacketContent = root; + + drive->fOutput(drive->fPacket); + + return YES; } -/// @brief Make a EPM+HeFS drive out of the disk. -/// @param drive The drive to write on. -/// @return If it was sucessful, see err_local_get(). -_Output Bool HeFileSystemParser::FormatGPT(_Input _Output DriveTrait* drive, - _Input const Lba end_lba, _Input const Int32 flags, - const Char* part_name) { - NE_UNUSED(drive); - NE_UNUSED(end_lba); - NE_UNUSED(flags); - NE_UNUSED(part_name); +Boolean fs_init_hefs(Void) noexcept { + kout << "Creating main disk...\r"; - return NO; + auto drv = io_construct_main_drive(); + + if (drv.fPacket.fPacketReadOnly == YES) + ke_panic(RUNTIME_CHECK_FILESYSTEM, "Main filesystem cannot be mounted."); + + HeFileSystemParser parser; + parser.Format(&drv, kHeFSEncodingUTF16, kHeFSDefaultVoluneName); + + return YES; } -} // namespace Kernel +} // namespace Kernel::HeFS #endif // ifdef __FSKIT_INCLUDES_HEFS__ diff --git a/dev/kernel/src/FS/NeFS.cc b/dev/kernel/src/FS/NeFS.cc index d0934f94..e0cdc71b 100644 --- a/dev/kernel/src/FS/NeFS.cc +++ b/dev/kernel/src/FS/NeFS.cc @@ -415,25 +415,14 @@ _Output NEFS_CATALOG_STRUCT* NeFileSystemParser::CreateCatalog(_Input const Char return nullptr; } -_Output Bool NeFileSystemParser::FormatGPT(_Input _Output DriveTrait* drive, - _Input const Lba end_lba, _Input const Int32 flags, - const Char* part_name) { - NE_UNUSED(drive); - NE_UNUSED(end_lba); - NE_UNUSED(flags); - NE_UNUSED(part_name); - - (Void)(kout << "FormatGPT: Not implemented yet.\r"); - - return NO; -} - /// @brief Make a EPM+NeFS drive out of the disk. /// @param drive The drive to write on. /// @return If it was sucessful, see err_global_get(). -bool NeFileSystemParser::FormatEPM(_Input _Output DriveTrait* drive, _Input const Lba endLba, - _Input const Int32 flags, const Char* part_name) { - if (*part_name == 0 || endLba == 0) return false; +bool NeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input const Int32 flags, + const Char* part_name) { + if (*part_name == 0) return false; + + NE_UNUSED(flags); // verify disk. drive->fVerify(drive->fPacket); @@ -449,7 +438,7 @@ bool NeFileSystemParser::FormatEPM(_Input _Output DriveTrait* drive, _Input cons Char fs_buf[sizeof(NEFS_ROOT_PARTITION_BLOCK)] = {0}; - Lba start = kNeFSRootCatalogStartAddress; + Lba start = drive->fLbaStart; drive->fPacket.fPacketContent = fs_buf; drive->fPacket.fPacketSize = sizeof(NEFS_ROOT_PARTITION_BLOCK); @@ -457,129 +446,48 @@ bool NeFileSystemParser::FormatEPM(_Input _Output DriveTrait* drive, _Input cons drive->fInput(drive->fPacket); - if (flags & kNeFSPartitionTypeBoot) { - // make it bootable when needed. - Char buf_epm[kNeFSSectorSz] = {0}; - - EPM_PART_BLOCK* epm_boot = (EPM_PART_BLOCK*) buf_epm; - - // Write a new EPM entry. - - constexpr auto kFsName = "NeFS"; - constexpr auto kBlockName = "NeKernel:"; - - epm_boot->FsVersion = kNeFSVersionInteger; - epm_boot->LbaStart = start; - epm_boot->SectorSz = kNeFSSectorSz; - - rt_copy_memory(reinterpret_cast(const_cast(kFsName)), epm_boot->Fs, - rt_string_len(kFsName)); - rt_copy_memory(reinterpret_cast(const_cast(kBlockName)), epm_boot->Name, - rt_string_len(kBlockName)); - rt_copy_memory(reinterpret_cast(const_cast(kEPMMagic)), epm_boot->Magic, - rt_string_len(kEPMMagic)); - - Lba outEpmLba = kEPMBootBlockLba; - - Char buf[kNeFSSectorSz] = {0}; - - Lba prevStart = 0; - SizeT cnt = 0; - - while (drive->fPacket.fPacketGood) { - drive->fPacket.fPacketContent = buf; - drive->fPacket.fPacketSize = sizeof(EPM_PART_BLOCK); - drive->fPacket.fPacketLba = outEpmLba; - - drive->fInput(drive->fPacket); - - if (buf[0] == 0) { - epm_boot->LbaStart = prevStart; - - if (epm_boot->LbaStart) epm_boot->LbaStart = outEpmLba; - - epm_boot->LbaEnd = endLba; - epm_boot->NumBlocks = cnt; - - drive->fPacket.fPacketContent = buf_epm; - drive->fPacket.fPacketSize = sizeof(EPM_PART_BLOCK); - drive->fPacket.fPacketLba = outEpmLba; - - drive->fOutput(drive->fPacket); + NEFS_ROOT_PARTITION_BLOCK* part_block = (NEFS_ROOT_PARTITION_BLOCK*) fs_buf; - break; - } else { - prevStart = ((EPM_PART_BLOCK*) buf)->LbaStart + ((EPM_PART_BLOCK*) buf)->LbaEnd; - } - - outEpmLba += sizeof(EPM_PART_BLOCK); - ++cnt; - } - } - - // disk isnt faulty and data has been fetched. - while (drive->fPacket.fPacketGood) { - NEFS_ROOT_PARTITION_BLOCK* part_block = (NEFS_ROOT_PARTITION_BLOCK*) fs_buf; - - // check for an empty partition here. - if (part_block->PartitionName[0] == 0 && - rt_string_cmp(part_block->Ident, kNeFSIdent, kNeFSIdentLen)) { - // partition is free and valid. - - part_block->Version = kNeFSVersionInteger; - - const auto kNeFSUntitledHD = part_name; - - rt_copy_memory((VoidPtr) kNeFSIdent, (VoidPtr) part_block->Ident, kNeFSIdentLen); - - rt_copy_memory((VoidPtr) kNeFSUntitledHD, (VoidPtr) part_block->PartitionName, - rt_string_len(kNeFSUntitledHD)); + const auto kNeFSUntitledHD = part_name; - SizeT sectorCount = drv_get_sector_count(); - SizeT diskSize = drv_get_size(); + rt_copy_memory((VoidPtr) kNeFSIdent, (VoidPtr) part_block->Ident, kNeFSIdentLen); - part_block->Kind = kNeFSPartitionTypeStandard; - part_block->StartCatalog = kNeFSCatalogStartAddress; - part_block->Flags = kNeFSPartitionTypeStandard; - part_block->CatalogCount = sectorCount / sizeof(NEFS_CATALOG_STRUCT); - part_block->FreeSectors = sectorCount / sizeof(NEFS_CATALOG_STRUCT); - part_block->SectorCount = sectorCount; - part_block->DiskSize = diskSize; - part_block->FreeCatalog = sectorCount / sizeof(NEFS_CATALOG_STRUCT); + rt_copy_memory((VoidPtr) kNeFSUntitledHD, (VoidPtr) part_block->PartitionName, + rt_string_len(kNeFSUntitledHD)); - drive->fPacket.fPacketContent = fs_buf; - drive->fPacket.fPacketSize = sizeof(NEFS_ROOT_PARTITION_BLOCK); - drive->fPacket.fPacketLba = kNeFSRootCatalogStartAddress; + SizeT sectorCount = drv_get_sector_count(); + SizeT diskSize = drv_get_size(); - drive->fOutput(drive->fPacket); + part_block->Version = kNeFSVersionInteger; - (Void)(kout << "drive kind: " << drive->fProtocol() << kendl); + part_block->Kind = kNeFSPartitionTypeStandard; + part_block->StartCatalog = start + sizeof(NEFS_ROOT_PARTITION_BLOCK); + part_block->Flags = 0UL; + part_block->CatalogCount = sectorCount / sizeof(NEFS_CATALOG_STRUCT); + part_block->FreeSectors = sectorCount / sizeof(NEFS_CATALOG_STRUCT); + part_block->SectorCount = sectorCount; + part_block->DiskSize = diskSize; + part_block->FreeCatalog = sectorCount / sizeof(NEFS_CATALOG_STRUCT); - (Void)(kout << "partition name: " << part_block->PartitionName << kendl); - (Void)(kout << "start: " << hex_number(part_block->StartCatalog) << kendl); - (Void)(kout << "number of catalogs: " << hex_number(part_block->CatalogCount) << kendl); - (Void)(kout << "free catalog: " << hex_number(part_block->FreeCatalog) << kendl); - (Void)(kout << "free sectors: " << hex_number(part_block->FreeSectors) << kendl); - (Void)(kout << "sector size: " << hex_number(part_block->SectorSize) << kendl); - - // write the root catalog. - this->CreateCatalog(kNeFSRoot, 0, kNeFSCatalogKindDir); - - return true; - } + drive->fPacket.fPacketContent = fs_buf; + drive->fPacket.fPacketSize = sizeof(NEFS_ROOT_PARTITION_BLOCK); + drive->fPacket.fPacketLba = start; - kout << "partition block already exists.\r"; + drive->fOutput(drive->fPacket); - start += part_block->DiskSize; + (Void)(kout << "drive kind: " << drive->fProtocol() << kendl); - drive->fPacket.fPacketContent = fs_buf; - drive->fPacket.fPacketSize = sizeof(NEFS_ROOT_PARTITION_BLOCK); - drive->fPacket.fPacketLba = start; + (Void)(kout << "partition name: " << part_block->PartitionName << kendl); + (Void)(kout << "start: " << hex_number(part_block->StartCatalog) << kendl); + (Void)(kout << "number of catalogs: " << hex_number(part_block->CatalogCount) << kendl); + (Void)(kout << "free catalog: " << hex_number(part_block->FreeCatalog) << kendl); + (Void)(kout << "free sectors: " << hex_number(part_block->FreeSectors) << kendl); + (Void)(kout << "sector size: " << hex_number(part_block->SectorSize) << kendl); - drive->fInput(drive->fPacket); - } + // write the root catalog. + this->CreateCatalog(kNeFSRoot, 0, kNeFSCatalogKindDir); - return false; + return true; } /// @brief Writes the data fork into a specific catalog. @@ -971,6 +879,9 @@ Boolean fs_init_nefs(Void) noexcept { if (kMountpoint.A().fPacket.fPacketReadOnly == YES) ke_panic(RUNTIME_CHECK_FILESYSTEM, "Main filesystem cannot be mounted."); + NeFileSystemParser parser; + parser.Format(&kMountpoint.A(), 0, kNeFSVolumeName); + return YES; } } // namespace Kernel::NeFS diff --git a/dev/kernel/src/WideUtils.cc b/dev/kernel/src/WideUtils.cc index e6c5a533..0f628e00 100644 --- a/dev/kernel/src/WideUtils.cc +++ b/dev/kernel/src/WideUtils.cc @@ -10,8 +10,24 @@ namespace Kernel { Size wrt_string_len(const Utf16Char* str) { SizeT len{0}; - while (str[len] != 0) ++len; + while (str[len] != u'\0') ++len; return len; } + +Int wrt_copy_memory(const voidPtr src, voidPtr dst, Size len) { + Utf16Char* srcChr = reinterpret_cast(src); + Utf16Char* dstChar = reinterpret_cast(dst); + + Size index = 0; + + while (index < len) { + dstChar[index] = srcChr[index]; + ++index; + } + + dstChar[index] = 0; + + return index; +} } // namespace Kernel \ No newline at end of file diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index d20dfa5e..c9f7886d 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -45,7 +45,7 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0100) \\ \verb|fSectorCount| & \verb|UInt64| & Total sector count \\ \verb|fSectorSize| & \verb|UInt64| & Size of each sector \\ -\verb|fDriveKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\ +\verb|fDiskKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\ \verb|fEncoding| & \verb|UInt8| & Encoding mode (UTF-8, UTF-16, etc.) \\ \verb|fStartIND| & \verb|UInt64| & Starting LBA of inode tree \\ \verb|fEndIND| & \verb|UInt64| & Ending LBA of inode tree \\ diff --git a/modules_pio_x64.sh b/modules_pio_x64.sh index 49f0e667..b091704e 100755 --- a/modules_pio_x64.sh +++ b/modules_pio_x64.sh @@ -5,7 +5,7 @@ # 04/05/25: Improve and fix script. cd dev/boot/modules/SysChk -btb amd64-pio-gpt.json +btb amd64-pio-epm.json cd ../ cd BootNet btb amd64.json \ No newline at end of file -- cgit v1.2.3 From e5efb02ff49e7a5a0083acc5f4b1af5fbb73b518 Mon Sep 17 00:00:00 2001 From: Amlal Date: Sun, 27 Apr 2025 13:12:12 +0200 Subject: dev, kernel: AHCI support and HeFS improvements, alongside scheduler allocation tree fix. Signed-off-by: Amlal --- dev/boot/BootKit/BootKit.h | 2 +- dev/kernel/FSKit/HeFS.h | 2 + dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc | 21 +-- dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc | 182 ++++++++++++++++-------- dev/kernel/StorageKit/DMA.h | 58 ++++++++ dev/kernel/src/BitMapMgr.cc | 2 - dev/kernel/src/FS/HeFS.cc | 139 ++++++++---------- dev/kernel/src/UserProcessScheduler.cc | 8 +- docs/tex/hefs.tex | 108 +++++++------- 9 files changed, 322 insertions(+), 200 deletions(-) create mode 100644 dev/kernel/StorageKit/DMA.h (limited to 'docs') diff --git a/dev/boot/BootKit/BootKit.h b/dev/boot/BootKit/BootKit.h index 95be7be8..aa7dde85 100644 --- a/dev/boot/BootKit/BootKit.h +++ b/dev/boot/BootKit/BootKit.h @@ -262,7 +262,7 @@ inline Boolean BDiskFormatFactory::Format(const Char* part_name) { /// @note also look at EPM headers, for free part blocks. (only applies if EPM or vEPM is used) if (fDiskDev.GetDiskSize() < kMinimumDiskSize) { - Boot::ThrowError(L"Drive-Too-Tiny", L"Can't format a NeFS partition here."); + Boot::ThrowError(L"Drive-Too-Tiny", L"Can't format a EPM partition here."); return false; } diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index c18f05d4..fd462b17 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -39,6 +39,7 @@ struct HEFS_BOOT_NODE; struct HEFS_INDEX_NODE; struct HEFS_INDEX_NODE_DIRECTORY; +struct HEFS_JOURNAL_NODE; enum { kHeFSHardDrive = 0xC0, // Hard Drive @@ -162,6 +163,7 @@ struct PACKED ALIGN(8) HEFS_INDEX_NODE final { }; enum { + kHeFSInvalidColor = 0, kHeFSRed = 100, kHeFSBlack, kHeFSColorCount, diff --git a/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc b/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc index 8e80c4d8..d7a8baf6 100644 --- a/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc +++ b/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc @@ -90,7 +90,7 @@ UIntPtr hal_get_phys_address(VoidPtr virt) { mmi_page_status((Detail::PTE*) pte); - return pte->PhysicalAddress; + return (pte->PhysicalAddress << 12) | (kVMAddr & 0xFFF); } /***********************************************************************************/ @@ -125,16 +125,21 @@ EXTERN_C Int32 mm_map_page(VoidPtr virtual_address, VoidPtr physical_address, UI UInt64* pt = reinterpret_cast(pde & ~kPageMask); Detail::PTE* pte = (Detail::PTE*) pt[(kVMAddr >> 12) & kMask9]; - pte->Present = !!(flags & kMMFlagsPresent); - pte->Wr = !!(flags & kMMFlagsWr); - pte->User = !!(flags & kMMFlagsUser); - pte->Nx = !!(flags & kMMFlagsNX); - pte->Pcd = !!(flags & kMMFlagsPCD); - pte->Pwt = !!(flags & kMMFlagsPwt); - pte->PhysicalAddress = ((UIntPtr) (physical_address)); + pte->Present = !!(flags & kMMFlagsPresent); + pte->Wr = !!(flags & kMMFlagsWr); + pte->User = !!(flags & kMMFlagsUser); + pte->Nx = !!(flags & kMMFlagsNX); + pte->Pcd = !!(flags & kMMFlagsPCD); + pte->Pwt = !!(flags & kMMFlagsPwt); + + pte->PhysicalAddress = ((UIntPtr) (physical_address)) >> 12; hal_invl_tlb(virtual_address); + asm volatile("clflush (%0)" : : "r"(virtual_address) : "memory"); + + asm volatile("mfence" ::: "memory"); + mmi_page_status(pte); return kErrorSuccess; diff --git a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc index aa9ab2cb..a28fa0ab 100644 --- a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc +++ b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -54,10 +55,11 @@ using namespace Kernel; STATIC PCI::Device kSATADev; STATIC HbaMemRef kSATAHba; -STATIC Lba kSATASectorCount = 0UL; -STATIC UInt16 kSATAIndex = 0U; -STATIC Char kCurrentDiskModel[50] = {"GENERIC SATA"}; -STATIC UInt16 kSATAPortsImplemented = 0U; +STATIC Lba kSATASectorCount = 0UL; +STATIC UInt16 kSATAIndex = 0U; +STATIC Char kCurrentDiskModel[50] = {"GENERIC SATA"}; +STATIC UInt16 kSATAPortsImplemented = 0U; +STATIC ALIGN(4096) UInt8 kIdentifyData[kAHCISectorSize] = {0}; template STATIC Void drv_std_input_output_ahci(UInt64 lba, UInt8* buffer, SizeT sector_sz, @@ -67,42 +69,37 @@ STATIC Int32 drv_find_cmd_slot_ahci(HbaPort* port) noexcept; STATIC Void drv_compute_disk_ahci() noexcept; +/// @brief Identify device and read LBA info, Disk OEM vendor... STATIC Void drv_compute_disk_ahci() noexcept { kSATASectorCount = 0UL; - /// Normally 512 bytes, but add an additional 512 bytes to make 1 KIB. - const UInt16 kSzIdent = 512; + rt_set_memory(kIdentifyData, 0, kAHCISectorSize); - /// Push it to the stack - UInt8* identify_data = new UInt8[kSzIdent]; + drv_std_input_output_ahci(0, kIdentifyData, kAHCISectorSize, kAHCISectorSize); - /// Send AHCI command for identification. - drv_std_input_output_ahci(0, identify_data, kAHCISectorSize, kSzIdent); + // --> Reinterpret the 512-byte buffer as an array of 256 UInt16 words + UInt16* identify_words = reinterpret_cast(kIdentifyData); /// Extract 48-bit LBA. - UInt64 lba48_sectors = 0; - lba48_sectors |= (UInt64) identify_data[100]; - lba48_sectors |= (UInt64) identify_data[101] << 16; - lba48_sectors |= (UInt64) identify_data[102] << 32; + UInt64 lba48_sectors = 0UL; + lba48_sectors |= (UInt64) identify_words[100]; + lba48_sectors |= (UInt64) identify_words[101] << 16; + lba48_sectors |= (UInt64) identify_words[102] << 32; - /// Now verify if lba48 if (lba48_sectors == 0) - kSATASectorCount = (identify_data[61] << 16) | identify_data[60]; + kSATASectorCount = (identify_words[61] << 16) | identify_words[60]; else kSATASectorCount = lba48_sectors; for (Int32 i = 0; i < 20; i++) { - kCurrentDiskModel[i * 2] = (identify_data[27 + i] >> 8) & 0xFF; - kCurrentDiskModel[i * 2 + 1] = identify_data[27 + i] & 0xFF; + kCurrentDiskModel[i * 2] = (identify_words[27 + i] >> 8) & 0xFF; + kCurrentDiskModel[i * 2 + 1] = identify_words[27 + i] & 0xFF; } kCurrentDiskModel[40] = '\0'; (Void)(kout << "SATA Sector Count: " << hex_number(kSATASectorCount) << kendl); (Void)(kout << "SATA Disk Model: " << kCurrentDiskModel << kendl); - - delete[] identify_data; - identify_data = nullptr; } /// @brief Finds a command slot for a HBA port. @@ -127,81 +124,108 @@ STATIC Int32 drv_find_cmd_slot_ahci(HbaPort* port) noexcept { template STATIC Void drv_std_input_output_ahci(UInt64 lba, UInt8* buffer, SizeT sector_sz, SizeT size_buffer) noexcept { - UIntPtr slot = 0UL; - - slot = drv_find_cmd_slot_ahci(&kSATAHba->Ports[kSATAIndex]); + NE_UNUSED(sector_sz); - if (slot == ~0UL) { + if (!buffer || size_buffer == 0) { + kout << "Invalid buffer for AHCI I/O.\r"; err_global_get() = kErrorDisk; return; } - if (size_buffer > mib_cast(4) || sector_sz > kAHCISectorSize) return; + UIntPtr slot = drv_find_cmd_slot_ahci(&kSATAHba->Ports[kSATAIndex]); - if (!Write) { - // Zero-memory the buffer field. - rt_set_memory(buffer, 0, size_buffer); + if (slot == ~0UL) { + kout << "No free command slot!\r"; + err_global_get() = kErrorDisk; + return; } - /// prepare command header. volatile HbaCmdHeader* command_header = - ((volatile HbaCmdHeader*) (((UInt64) kSATAHba->Ports[kSATAIndex].Clb))); - - /// Offset to specific command slot. + (volatile HbaCmdHeader*) ((UInt64) kSATAHba->Ports[kSATAIndex].Clb); command_header += slot; - /// check for command header. MUST_PASS(command_header); - command_header->Struc.Cfl = sizeof(FisRegH2D) / sizeof(UInt32); - command_header->Struc.Write = Write; - command_header->Prdtl = 8; + // Clear old command table memory + volatile HbaCmdTbl* command_table = + (volatile HbaCmdTbl*) (((UInt64) command_header->Ctbau << 32) | command_header->Ctba); + rt_set_memory((VoidPtr) command_table, 0, sizeof(HbaCmdTbl)); - auto ctba_phys = ((UInt64) command_header->Ctbau << 32) | command_header->Ctba; - auto command_table = reinterpret_cast(ctba_phys); + VoidPtr ptr = Kernel::rtl_dma_alloc(size_buffer, 4096); - MUST_PASS(command_table); - - UIntPtr buffer_phys = HAL::hal_get_phys_address(buffer); + // Build the PRDT SizeT bytes_remaining = size_buffer; + SizeT prdt_index = 0; + UIntPtr buffer_phys = (UIntPtr) ptr; + + while (bytes_remaining > 0 && prdt_index < 8) { + SizeT chunk_size = bytes_remaining; + if (chunk_size > 8 * 1024) // AHCI recommends ~8 KiB per PRDT + chunk_size = 8 * 1024; + + command_table->Prdt[prdt_index].Dba = (UInt32) (buffer_phys & 0xFFFFFFFF); + command_table->Prdt[prdt_index].Dbau = (UInt32) (buffer_phys >> 32); + command_table->Prdt[prdt_index].Dbc = (UInt32) (chunk_size - 1); + command_table->Prdt[prdt_index].Ie = NO; + + buffer_phys += chunk_size; + bytes_remaining -= chunk_size; + ++prdt_index; + } - command_table->Prdt[0].Dba = (UInt32) (buffer_phys & 0xFFFFFFFF); - command_table->Prdt[0].Dbau = (UInt32) (buffer_phys >> 32); - command_table->Prdt[0].Dbc = bytes_remaining - 1; - command_table->Prdt[0].Ie = NO; + if (bytes_remaining > 0) { + kout << "Warning: AHCI PRDT overflow, cannot map full buffer.\r"; + err_global_get() = kErrorDisk; + return; + } + + command_header->Prdtl = prdt_index; + command_header->Struc.Cfl = sizeof(FisRegH2D) / sizeof(UInt32); + command_header->Struc.Write = Write; volatile FisRegH2D* h2d_fis = (volatile FisRegH2D*) (&command_table->Cfis[0]); h2d_fis->FisType = kFISTypeRegH2D; h2d_fis->CmdOrCtrl = CommandOrCTRL; h2d_fis->Command = - (Identify ? (kAHCICmdIdentify) : (Write ? kAHCICmdWriteDmaEx : kAHCICmdReadDmaEx)); + (Identify ? kAHCICmdIdentify : (Write ? kAHCICmdWriteDmaEx : kAHCICmdReadDmaEx)); - h2d_fis->Lba0 = (lba) & 0xFF; + h2d_fis->Lba0 = (lba >> 0) & 0xFF; h2d_fis->Lba1 = (lba >> 8) & 0xFF; h2d_fis->Lba2 = (lba >> 16) & 0xFF; - - h2d_fis->Device = kSATALBAMode; - h2d_fis->Lba3 = (lba >> 24) & 0xFF; h2d_fis->Lba4 = (lba >> 32) & 0xFF; h2d_fis->Lba5 = (lba >> 40) & 0xFF; - h2d_fis->CountLow = (size_buffer) & 0xFF; - h2d_fis->CountHigh = (size_buffer >> 8) & 0xFF; + if (Identify) { + h2d_fis->Device = 0; // DO NOT set LBAMODE flag + h2d_fis->CountLow = 1; // IDENTIFY always transfers 1 sector + h2d_fis->CountHigh = 0; + } else { + h2d_fis->Device = kSATALBAMode; + h2d_fis->CountLow = (size_buffer / kAHCISectorSize) & 0xFF; + h2d_fis->CountHigh = ((size_buffer / kAHCISectorSize) >> 8) & 0xFF; + } + + rtl_dma_flush(ptr, size_buffer); + // Issue command kSATAHba->Ports[kSATAIndex].Ci = (1 << slot); - for (Int32 i = 0; i < 1000000; ++i) { + while (YES) { if (!(kSATAHba->Ports[kSATAIndex].Ci & (1 << slot))) break; } + rtl_dma_flush(ptr, size_buffer); + if (kSATAHba->Is & kHBAErrTaskFile) { + kout << "AHCI Task File Error during I/O.\r"; err_global_get() = kErrorDiskIsCorrupted; return; + } else { + rt_copy_memory(ptr, buffer, size_buffer); + err_global_get() = kErrorSuccess; } - - err_global_get() = kErrorSuccess; } /*** @@ -251,6 +275,47 @@ STATIC BOOL ahci_enable_and_probe() { return YES; } +STATIC Bool drv_init_command_structures_ahci() { + // Allocate 4KiB for Command List (32 headers) + VoidPtr clb_mem = Kernel::rtl_dma_alloc(4096, 1024); + if (!clb_mem) { + kout << "Failed to allocate CLB memory!\r"; + return NO; + } + + UIntPtr clb_phys = HAL::hal_get_phys_address(clb_mem); + + kSATAHba->Ports[kSATAIndex].Clb = (UInt32)(clb_phys & 0xFFFFFFFF); + kSATAHba->Ports[kSATAIndex].Clbu = (UInt32)(clb_phys >> 32); + + // Clear it + rt_set_memory(clb_mem, 0, 4096); + + // For each command slot (up to 32) + volatile HbaCmdHeader* header = (volatile HbaCmdHeader*) clb_mem; + + for (Int32 i = 0; i < 32; ++i) { + // Allocate 4KiB for Command Table + VoidPtr ct_mem = Kernel::rtl_dma_alloc(4096, 128); + if (!ct_mem) { + kout << "Failed to allocate CTB memory for slot " << hex_number(i); + kout << "!\r"; + return NO; + } + + UIntPtr ct_phys = HAL::hal_get_phys_address(ct_mem); + + header[i].Ctba = (UInt32)(ct_phys & 0xFFFFFFFF); + header[i].Ctbau = (UInt32)(ct_phys >> 32); + + // Clear the command table + rt_set_memory((VoidPtr) ct_mem, 0, 4096); + } + + return YES; +} + + /// @brief Initializes an AHCI disk. /// @param pi the amount of ports that have been detected. /// @param atapi reference value, tells whether we should detect ATAPI instead of SATA. @@ -295,6 +360,11 @@ STATIC Bool drv_std_init_ahci(UInt16& pi, BOOL& atapi) { kSATAIndex = ahci_index; kSATAHba = mem_ahci; + if (!drv_init_command_structures_ahci()) { + err_global_get() = kErrorDisk; + return NO; + } + goto success_hba_fetch; } diff --git a/dev/kernel/StorageKit/DMA.h b/dev/kernel/StorageKit/DMA.h new file mode 100644 index 00000000..953683eb --- /dev/null +++ b/dev/kernel/StorageKit/DMA.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Amlal El Mahrouss. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include + +#ifdef __NE_AMD64__ +#define DMA_POOL_START (0x1000000) +#define DMA_POOL_SIZE (0x1000000) + +namespace Kernel { +inline UInt8* kDmaPoolPtr = (UInt8*) DMA_POOL_START; + +inline VoidPtr rtl_dma_alloc(SizeT size, SizeT align) { + UIntPtr addr = (UIntPtr) kDmaPoolPtr; + + addr = (addr + (align - 1)) & ~(align - 1); // Align up + + if (addr + size >= DMA_POOL_START + DMA_POOL_SIZE) { + kout << "DMA Pool exhausted!\r"; + + return nullptr; + } + + kDmaPoolPtr = (UInt8*) (addr + size); + return (VoidPtr) addr; +} + +inline Void rtl_dma_flush(Void* ptr, SizeT size_buffer) { + for (SizeT i = 0; i < size_buffer; ++i) { + asm volatile("clflush (%0)" : : "r"((UInt8*) ptr + i) : "memory"); + } + asm volatile("mfence" ::: "memory"); +} +} // namespace Kernel +#else +#endif \ No newline at end of file diff --git a/dev/kernel/src/BitMapMgr.cc b/dev/kernel/src/BitMapMgr.cc index 382d0be5..cc5797b3 100644 --- a/dev/kernel/src/BitMapMgr.cc +++ b/dev/kernel/src/BitMapMgr.cc @@ -62,8 +62,6 @@ namespace HAL { if (user) flags |= kMMFlagsUser; - flags |= HAL::kMMFlagsPCD; - return flags; } diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index d7cce9d6..eb6e6fc1 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -79,8 +79,7 @@ namespace Detail { NE_UNUSED(node); if (!dir || !node) { - ke_panic(RUNTIME_CHECK_FILESYSTEM, - "Error: Invalid directory node/boot_node in RB-Tree traversal."); + return; } if (dir->fChild != 0) { @@ -92,7 +91,7 @@ namespace Detail { } else if (dir->fPrev != 0) { start = dir->fPrev; } else { - start = node->fStartIND; + start = 0; } } @@ -218,7 +217,7 @@ namespace Detail { auto start = root->fStartIND; auto end = root->fEndIND; - auto hop_watch = 0; + auto hop_watch = 0UL; while (YES) { if (start > end) { @@ -226,16 +225,11 @@ namespace Detail { break; } - if (hop_watch > 100) { + if (hop_watch++ > (root->fEndIND / sizeof(HEFS_INDEX_NODE_DIRECTORY))) { kout << "Error: Hop watch exceeded, filesystem is stalling.\r"; break; } - if (start == 0) { - ++hop_watch; - start = root->fStartIND; - } - mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); mnt->fPacket.fPacketContent = dir; @@ -316,11 +310,11 @@ namespace Detail { return NO; } - if (tmpdir->fDeleted || !tmpdir->fCreated) { - dir->fChild = tmpdir->fChild; + if (tmpdir->fDeleted) { + dir->fChild = tmpdir->fChild; dir->fParent = tmpdir->fParent; - dir->fNext = tmpdir->fNext; - dir->fPrev = tmpdir->fPrev; + dir->fNext = tmpdir->fNext; + dir->fPrev = tmpdir->fPrev; mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); @@ -331,7 +325,9 @@ namespace Detail { return YES; } + auto old_start = start; hefsi_traverse_tree(tmpdir, root, start); + if (start == 0 || start == old_start) break; } return YES; @@ -349,7 +345,7 @@ namespace Detail { auto start = root->fStartIND; auto end = root->fEndIND; - auto hop_watch = 0; + auto hop_watch = 0UL; while (YES) { if (start > end) { @@ -357,16 +353,11 @@ namespace Detail { break; } - if (hop_watch > 100) { + if (hop_watch++ > (root->fEndIND / sizeof(HEFS_INDEX_NODE_DIRECTORY))) { kout << "Error: Hop watch exceeded, filesystem is stalling.\r"; break; } - if (start == 0) { - ++hop_watch; - start = root->fStartIND; - } - mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); mnt->fPacket.fPacketContent = dir; @@ -390,7 +381,9 @@ namespace Detail { } } + auto old_start = start; hefsi_traverse_tree(dir, root, start); + if (start == 0 || start == old_start) break; } delete dir; @@ -427,7 +420,7 @@ namespace Detail { auto start_cnt = 0UL; - auto hop_watch = 0; + auto hop_watch = 0UL; while (YES) { if (start > end) { @@ -435,16 +428,11 @@ namespace Detail { break; } - if (hop_watch > 100) { + if (hop_watch++ > (root->fEndIND / sizeof(HEFS_INDEX_NODE_DIRECTORY))) { kout << "Error: Hop watch exceeded, filesystem is stalling.\r"; break; } - if (start == 0) { - ++hop_watch; - start = root->fStartIND; - } - mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); mnt->fPacket.fPacketContent = dir; @@ -507,7 +495,9 @@ namespace Detail { } } + auto old_start = start; hefsi_traverse_tree(dir, root, start); + if (start == 0 || start == old_start) break; } delete dir; @@ -539,19 +529,16 @@ namespace Detail { auto start = root->fStartIND; - auto hop_watch = 0; + auto hop_watch = 0UL; while (YES) { - if (hop_watch > 100) { + if (start > root->fEndIND) break; + + if (hop_watch++ > (root->fEndIND / sizeof(HEFS_INDEX_NODE_DIRECTORY))) { kout << "Error: Hop watch exceeded, filesystem is stalling.\r"; break; } - if (start == 0) { - ++hop_watch; - start = root->fStartIND; - } - mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); mnt->fPacket.fPacketContent = dir; @@ -590,7 +577,11 @@ namespace Detail { } } + auto old_start = start; + hefsi_traverse_tree(dir, root, start); + + if (start == 0 || start == old_start) break; } delete dir; @@ -614,9 +605,10 @@ namespace Detail { auto start = root->fStartIND; + SizeT hop_watch = 0UL; + while (YES) { - if (start > root->fEndIND) - break; + if (start > root->fEndIND) break; mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); @@ -624,6 +616,11 @@ namespace Detail { mnt->fInput(mnt->fPacket); + if (hop_watch++ > (root->fSectorCount)) { // <-- NEW (stall protection) + kout << "Warning: Traversal stalled during balancing.\r"; + break; // Exit cleanly instead of hanging forever + } + if (!mnt->fPacket.fPacketGood) { delete dir; dir = nullptr; @@ -700,14 +697,21 @@ namespace Detail { return NO; } + auto old_start = start; hefsi_traverse_tree(dir, root, start); + if (start == 0 || start == old_start) break; + continue; } else { if (dir_parent->fNext == start) { hefsi_rotate_left(dir, start, mnt); + + auto old_start = start; hefsi_traverse_tree(dir, root, start); + if (start == 0 || start == old_start) break; + continue; } @@ -729,12 +733,12 @@ namespace Detail { } hefsi_rotate_right(dir, start, mnt); - hefsi_traverse_tree(dir, root, start); - - continue; } + auto old_start = start; hefsi_traverse_tree(dir, root, start); + + if (start == 0 || start == old_start) break; } delete dir; @@ -852,8 +856,6 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input drive->fPacket.fPacketSize = sizeof(HEFS_BOOT_NODE); drive->fPacket.fPacketContent = root; - HEFS_BOOT_NODE root_cpy = *root; - drive->fOutput(drive->fPacket); if (!drive->fPacket.fPacketGood) { @@ -865,20 +867,20 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input return NO; } - SizeT cnt = kHeFSBlockCount; + /// @note this allocates 4 ind at format. + SizeT cnt = 4UL; - Lba next = root_cpy.fStartIND; - Lba prev = next; + Lba next = drive->fLbaStart + sizeof(HEFS_BOOT_NODE) + sizeof(HEFS_BOOT_NODE); HEFS_INDEX_NODE_DIRECTORY* index_node = new HEFS_INDEX_NODE_DIRECTORY(); - + // Pre-allocate index node directory tree for (SizeT i = 0; i < cnt; ++i) { rt_set_memory(index_node, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); wrt_copy_memory((VoidPtr) u"/$", index_node->fName, wrt_string_len(u"/$")); - index_node->fFlags = flags; - index_node->fKind = kHeFSFileKindDirectory; + index_node->fFlags = flags; + index_node->fKind = kHeFSFileKindDirectory; index_node->fDeleted = kHeFSTimeMax; @@ -890,23 +892,25 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input index_node->fUID = 0; index_node->fGID = 0; index_node->fMode = 0; - index_node->fColor = kHeFSBlack; - index_node->fParent = next; + index_node->fColor = kHeFSBlack; index_node->fChild = 0; - index_node->fNext = next + sizeof(HEFS_INDEX_NODE_DIRECTORY); - index_node->fPrev = prev; + index_node->fParent = 0; + index_node->fNext = 0; + index_node->fPrev = 0; - drive->fPacket.fPacketLba = next; - drive->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); + drive->fPacket.fPacketLba = next; + drive->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); drive->fPacket.fPacketContent = index_node; - prev = next; next += sizeof(HEFS_INDEX_NODE_DIRECTORY); drive->fOutput(drive->fPacket); } + delete index_node; + index_node = nullptr; + // Create the directories, something UNIX inspired but more explicit and forward looking. this->CreateDirectory(drive, kHeFSEncodingUTF16, u"/boot"); @@ -921,7 +925,7 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input this->CreateFile(drive, kHeFSEncodingBinary, u"/", u".hefs"); delete root; - root = nullptr; + root = nullptr; if (drive->fPacket.fPacketGood) return YES; @@ -967,37 +971,18 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu Detail::hefsi_balance_filesystem(root, drive); - auto dirent = Detail::hefs_fetch_index_node_directory(root, drive, dir); - - if (dirent) { - kout << "Error: Directory already exists.\r"; - - delete dirent; - dirent = nullptr; - - delete node; - delete root; - - return NO; - } - - dirent = new HEFS_INDEX_NODE_DIRECTORY(); + auto dirent = new HEFS_INDEX_NODE_DIRECTORY(); rt_set_memory(dirent, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); wrt_copy_memory((VoidPtr) dir, dirent->fName, wrt_string_len(dir)); dirent->fAccessed = 0; - dirent->fCreated = 0; /// TODO: Add the current time. + dirent->fCreated = kHeFSTimeMax; /// TODO: Add the current time. dirent->fDeleted = 0; dirent->fModified = 0; dirent->fEntryCount = 0; - dirent->fParent = root->fStartIND; // No parent (it's the real root) - dirent->fChild = root->fEndIND; // No children yet - dirent->fNext = 0; // No next - dirent->fPrev = 0; // No previous - dirent->fKind = kHeFSFileKindDirectory; dirent->fFlags = flags; dirent->fChecksum = 0; diff --git a/dev/kernel/src/UserProcessScheduler.cc b/dev/kernel/src/UserProcessScheduler.cc index 5d983f34..96938d10 100644 --- a/dev/kernel/src/UserProcessScheduler.cc +++ b/dev/kernel/src/UserProcessScheduler.cc @@ -108,11 +108,13 @@ STATIC USER_PROCESS::USER_HEAP_TREE* sched_try_go_upper_heap_tree( tree = tree->MemoryParent; if (tree) { - tree = tree->MemoryParent; + auto tree_tmp = tree->MemoryNext; - if (!tree) { - return nullptr; + if (!tree_tmp) { + return tree; } + + return tree_tmp; } return tree; diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index c9f7886d..75f98bcc 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -24,16 +24,16 @@ showstringspaces=false } -\title{HeFS (Hierarchical Embedded File System) Specification} +\title{HeFS (High-Throughput Extended File System) Specification} \author{Amlal El Mahrouss} -\date{2024–2025} +\date{2024--2025} \begin{document} \maketitle \section{Overview} -HeFS is a custom filesystem developed as part of the NeKernel project. It offers a journaling-like inode tree structure using red-black tree-inspired navigation for directories. Designed for robust use in desktop and server workloads, it supports various encoding modes and drive types. +HeFS is a custom filesystem developed as part of the NeKernel project. It offers a journaling-like inode tree structure using red-black tree-inspired navigation for directories. Designed for robust use in desktop, server, and embedded environments, it supports various encoding modes and drive types. \section{Boot Node Structure} \begin{longtable}{|l|l|p{8cm}|} @@ -42,39 +42,48 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \hline \verb|fMagic| & \verb|char[8]| & Filesystem magic (" HeFS") \\ \verb|fVolName| & \verb|Utf16Char[128]| & Volume name \\ -\verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0100) \\ +\verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0101) \\ +\verb|fBadSectors| & \verb|UInt64| & Number of bad sectors detected \\ \verb|fSectorCount| & \verb|UInt64| & Total sector count \\ \verb|fSectorSize| & \verb|UInt64| & Size of each sector \\ +\verb|fChecksum| & \verb|UInt32| & CRC32 checksum of the boot node \\ \verb|fDiskKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\ \verb|fEncoding| & \verb|UInt8| & Encoding mode (UTF-8, UTF-16, etc.) \\ \verb|fStartIND| & \verb|UInt64| & Starting LBA of inode tree \\ \verb|fEndIND| & \verb|UInt64| & Ending LBA of inode tree \\ -\verb|fChecksum| & \verb|UInt32| & CRC32 of boot node \\ +\verb|fINDCount| & \verb|UInt64| & Number of directory nodes allocated \\ \verb|fDiskSize| & \verb|UInt64| & Logical size of the disk \\ +\verb|fDiskStatus| & \verb|UInt16| & Status of the disk (e.g., unlocked, locked) \\ +\verb|fDiskFlags| & \verb|UInt16| & Disk flags (e.g., read-only) \\ +\verb|fVID| & \verb|UInt16| & Virtual ID (EPM integration) \\ \hline \end{longtable} \section{File Types and Flags} \subsection*{File Kinds} \begin{itemize}[label=--] - \item \verb|0x00| — Regular File - \item \verb|0x01| — Directory - \item \verb|0x02| — Block Device - \item \verb|0x03| — Character Device - \item \verb|0x04| — FIFO - \item \verb|0x05| — Socket - \item \verb|0x06| — Symbolic Link + \item \verb|0x00| --- Regular File + \item \verb|0x01| --- Directory + \item \verb|0x02| --- Block Device + \item \verb|0x03| --- Character Device + \item \verb|0x04| --- FIFO + \item \verb|0x05| --- Socket + \item \verb|0x06| --- Symbolic Link + \item \verb|0x07| --- Unknown \end{itemize} \subsection*{Drive Types} \begin{itemize}[label=--] - \item \verb|0xC0| — Hard Drive - \item \verb|0xC1| — SSD - \item \verb|0xCC| — Mass Storage Device (USB) + \item \verb|0xC0| --- Hard Drive + \item \verb|0xC1| --- Solid State Drive + \item \verb|0x0C| --- Optical Drive + \item \verb|0xCC| --- USB Mass Storage + \item \verb|0xC4| --- SCSI/SAS Drive + \item \verb|0xC6| --- Flash Drive \end{itemize} \section{Index Node Structure} -The inode tree allows for fast access and recovery via dual start/end block mappings and timestamped metadata. +Files are stored through block links, offering native recovery fields and MIME type support. \begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE structure}] struct HEFS_INDEX_NODE { @@ -83,6 +92,8 @@ struct HEFS_INDEX_NODE { UInt16 fKind; UInt32 fSize; UInt32 fChecksum, fRecoverChecksum, fBlockChecksum, fLinkChecksum; + Utf16Char fMime[256]; + Boolean fSymLink; ATime fCreated, fAccessed, fModified, fDeleted; UInt32 fUID, fGID; UInt32 fMode; @@ -93,59 +104,50 @@ struct HEFS_INDEX_NODE { \end{lstlisting} \section{Directory Node Structure} -Each directory is a red-black tree node with child and sibling pointers. +Directories are organized into a red-black tree for efficient balancing. \begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE\_DIRECTORY structure}] struct HEFS_INDEX_NODE_DIRECTORY { Utf16Char fName[256]; - UInt32 fFlags; - UInt16 fKind; - UInt32 fSize; - UInt32 fChecksum, fIndexNodeChecksum; - ATime fCreated, fAccessed, fModified, fDeleted; - UInt32 fUID, fGID; - UInt32 fMode; - UInt64 fIndexNodeStart[16], fIndexNodeEnd[16]; - UInt8 fColor; - Lba fNext, fPrev, fChild, fParent; + UInt32 fFlags; + UInt16 fKind; + UInt32 fEntryCount; + UInt32 fChecksum, fIndexNodeChecksum; + Utf16Char fDim[256]; + ATime fCreated, fAccessed, fModified, fDeleted; + UInt32 fUID, fGID; + UInt32 fMode; + UInt64 fIndexNodeStart[16], fIndexNodeEnd[16]; + UInt8 fColor; + Lba fNext, fPrev, fChild, fParent; }; \end{lstlisting} \section{Filesystem Design} -HeFS is designed to be robust and efficient, with a focus on: +HeFS is designed with the following objectives: \begin{itemize} - \item Fast access to files and directories - \item Efficient use of disk space - \item Support for large files and directories - \item Compatibility with various drive types - \item Easy recovery from errors - \item Support for advanced features like journaling and permissions + \item Red-black tree navigation for efficient directory balancing + \item Journaling fields for block-level recovery + \item Multi-encoding support: UTF-8, UTF-16, UTF-32 + \item Advanced MIME type support + \item Redundant fields (checksums, recovery inodes) for crash resistance + \item Extensible for future LVM (Logical Volume Management) and network filesystem support \end{itemize} -The filesystem uses a red-black tree structure for directories, allowing for efficient insertion, deletion, and searching of files. -Each file and directory is represented by an index node directory, which contains metadata such as timestamps, permissions, and checksums. - -The index node directory itself contains index nodes, which are the actual data structures that store file and directory information. -The filesystem also includes a boot node, which contains information about the filesystem itself, such as the magic number, version, and size. - -The filesystem is designed to be extensible, allowing for future enhancements and features to be added without breaking compatibility with existing data. - \section{Minimum Requirements} -HeFS requires at least 4GB of free space on the drive to function properly. -It is recommended to have more than 8GB of free space for optimal performance. -The filesystem is designed to work with various drive types, including hard drives, SSDs, and USB mass storage devices. - -But that really depends on what you do with the filesystem. +HeFS expects a minimum disk size of 4GB. Optimal performance is achieved with 8GB or more. +Supports: HDDs, SSDs, USB mass storage, SCSI/SAS, optical drives. \section{Future Work} -Planned extensions include: +Planned enhancements include: \begin{itemize} - \item Journaling recovery logic - \item Advanced permission flags - \item Logical volume management support - \item Cross-filesystem linking (EPM, and GPT integration) + \item Full journaling implementation (recovery on crash) + \item Advanced ACLs (Access Control Lists) and permissions + \item Logical Volume Management (LVM) integration + \item Backup Superblock and dual-boot sectors + \item Online filesystem checking and self-healing algorithms \end{itemize} -\end{document} \ No newline at end of file +\end{document} -- cgit v1.2.3 From 2ead335ccc7afd8e1b2a6533e966c10f49fbdfe9 Mon Sep 17 00:00:00 2001 From: Amlal Date: Mon, 28 Apr 2025 09:06:35 +0200 Subject: dev, kernel: HeFS, Tooling: reworked the mkfs.hefs tool for the new filesystem. other/related: - Add new KPC codes. - Final refactors for HeFS's Format method. - Dma pool improvements. - Better standard disk I/O names. - Add mm_memory_fence function inside HalPagingMgrAMD64.cc Signed-off-by: Amlal --- dev/boot/BootKit/HW/SATA.h | 2 +- dev/kernel/FSKit/HeFS.h | 36 ++++++------- dev/kernel/HALKit/AMD64/HalKernelMain.cc | 2 +- dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc | 16 ++++-- dev/kernel/HALKit/AMD64/Processor.h | 2 + dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc | 14 +++--- dev/kernel/HALKit/AMD64/Storage/DMA+Generic.cc | 6 +-- dev/kernel/HALKit/AMD64/Storage/PIO+Generic.cc | 4 +- dev/kernel/KernelKit/FileMgr.h | 2 +- dev/kernel/KernelKit/KPC.h | 6 ++- dev/kernel/NewKit/KString.h | 2 +- dev/kernel/StorageKit/DmaPool.h | 21 ++++++-- dev/kernel/src/DriveMgr.cc | 2 +- dev/kernel/src/FS/HeFS+FileMgr.cc | 4 +- dev/kernel/src/FS/HeFS.cc | 67 ++++++++++++++----------- dev/kernel/src/FS/NeFS.cc | 8 +-- dev/kernel/src/Utils.cc | 1 - dev/modules/AHCI/AHCI.h | 4 +- dev/modules/ATA/ATA.h | 4 +- dev/user/ProcessCodes.h | 1 + docs/tex/hefs.tex | 2 +- tooling/hefs.h | 67 +++++++++++-------------- tooling/mkfs.hefs.cc | 24 ++++----- 23 files changed, 161 insertions(+), 136 deletions(-) (limited to 'docs') diff --git a/dev/boot/BootKit/HW/SATA.h b/dev/boot/BootKit/HW/SATA.h index 5c70c18c..8cf48ec7 100644 --- a/dev/boot/BootKit/HW/SATA.h +++ b/dev/boot/BootKit/HW/SATA.h @@ -28,7 +28,7 @@ class BootDeviceSATA final { operator bool() { return this->Leak().mDetected; } - SizeT GetDiskSize() { return drv_get_size(); } + SizeT GetDiskSize() { return drv_std_get_size(); } constexpr static auto kSectorSize = kAHCISectorSize; diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index 90a2e274..515034fc 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -28,12 +28,6 @@ #define kHeFSDefaultVoluneName u8"HeFS Volume" -#define kHeFSDIMBootDir u8"boot-x/dir" -#define kHeFSMIMEBootFile u8"boot-x/file" - -#define kHeFSDIMSystemDir u8"system-x/dir" -#define kHeFSMIMESystemFile u8"system-x/file" - #define kHeFSSearchAllStr u8"*" struct HEFS_BOOT_NODE; @@ -101,19 +95,19 @@ typedef UInt64 ATime; /// @details Acts like a superblock, it contains the information about the filesystem. /// @note The boot node is the first block of the filesystem. struct PACKED HEFS_BOOT_NODE final { - Kernel::Char fMagic[kHeFSMagicLen]; /// @brief Magic number of the filesystem. + Kernel::Char fMagic[kHeFSMagicLen]; /// @brief Magic number of the filesystem. Kernel::Utf8Char fVolName[kHeFSPartNameLen]; /// @brief Volume name. - Kernel::UInt32 fVersion; /// @brief Version of the filesystem. - Kernel::UInt64 fBadSectors; /// @brief Number of bad sectors in the filesystem. - Kernel::UInt64 fSectorCount; /// @brief Number of sectors in the filesystem. - Kernel::UInt64 fSectorSize; /// @brief Size of the sector. - Kernel::UInt32 fChecksum; /// @brief Checksum of the boot node. - Kernel::UInt8 fDiskKind; /// @brief Kind of the drive. (Hard Drive, Solid State Drive, Optical - /// Drive, etc). - Kernel::UInt8 fEncoding; /// @brief Encoding of the filesystem. (UTF-8, UTF-16, etc). - Kernel::UInt64 fStartIND; /// @brief Start of the INode tree. - Kernel::UInt64 fEndIND; /// @brief End of the INode tree. - Kernel::UInt64 fINDCount; /// @brief Number of leafs in the INode tree. + Kernel::UInt32 fVersion; /// @brief Version of the filesystem. + Kernel::UInt64 fBadSectors; /// @brief Number of bad sectors in the filesystem. + Kernel::UInt64 fSectorCount; /// @brief Number of sectors in the filesystem. + Kernel::UInt64 fSectorSize; /// @brief Size of the sector. + Kernel::UInt32 fChecksum; /// @brief Checksum of the boot node. + Kernel::UInt8 fDiskKind; /// @brief Kind of the drive. (Hard Drive, Solid State Drive, Optical + /// Drive, etc). + Kernel::UInt8 fEncoding; /// @brief Encoding of the filesystem. (UTF-8, UTF-16, etc). + Kernel::UInt64 fStartIND; /// @brief Start of the INode tree. + Kernel::UInt64 fEndIND; /// @brief End of the INode tree. + Kernel::UInt64 fINDCount; /// @brief Number of leafs in the INode tree. Kernel::UInt64 fDiskSize; /// @brief Size of the disk. (Could be a virtual size, that is not the /// real size of the disk.) Kernel::UInt16 fDiskStatus; /// @brief Status of the disk. (locked, unlocked, error, invalid). @@ -124,7 +118,7 @@ struct PACKED HEFS_BOOT_NODE final { Kernel::UInt64 fReserved2; /// @brief Reserved for future use. Kernel::UInt64 fReserved3; /// @brief Reserved for future use. Kernel::UInt64 fReserved4; /// @brief Reserved for future use. - Kernel::Char fPad[272]; + Kernel::Char fPad[272]; }; inline constexpr Kernel::ATime kHeFSTimeInvalid = 0x0000000000000000; @@ -136,11 +130,11 @@ inline constexpr Kernel::ATime kHeFSTimeMax = 0xFFFFFFFFFFFFFFFF - 1; /// @note The index node is used to store the file information of a file. struct PACKED ALIGN(8) HEFS_INDEX_NODE final { Kernel::Utf8Char fName[kHeFSFileNameLen]; /// @brief File name. - Kernel::UInt32 fFlags; /// @brief File flags. + Kernel::UInt32 fFlags; /// @brief File flags. Kernel::UInt16 fKind; /// @brief File kind. (Regular, Directory, Block, Character, FIFO, Socket, /// Symbolic Link, Unknown). Kernel::UInt32 fSize; /// @brief File size. - Kernel::UInt32 fChecksum; /// @brief Checksum. + Kernel::UInt32 fChecksum; /// @brief Checksum. Kernel::Boolean fSymLink; /// @brief Is this a symbolic link? (if yes, the fName is the path to /// the file and blocklinkstart and end contains it's inodes.) diff --git a/dev/kernel/HALKit/AMD64/HalKernelMain.cc b/dev/kernel/HALKit/AMD64/HalKernelMain.cc index 66793aa6..a6bf09c7 100644 --- a/dev/kernel/HALKit/AMD64/HalKernelMain.cc +++ b/dev/kernel/HALKit/AMD64/HalKernelMain.cc @@ -142,4 +142,4 @@ EXTERN_C Kernel::Void hal_real_init(Kernel::Void) noexcept { ++team_index; } } -#endif // ifndef __NE_MODULAR_KERNEL_COMPONENTS__ \ No newline at end of file +#endif // ifndef __NE_MODULAR_KERNEL_COMPONENTS__ \ No newline at end of file diff --git a/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc b/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc index d7a8baf6..b6176e39 100644 --- a/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc +++ b/dev/kernel/HALKit/AMD64/HalPagingMgrAMD64.cc @@ -93,6 +93,18 @@ UIntPtr hal_get_phys_address(VoidPtr virt) { return (pte->PhysicalAddress << 12) | (kVMAddr & 0xFFF); } +/***********************************************************************************/ +/// @brief clflush+mfence helper function. +/***********************************************************************************/ +EXTERN_C Int32 mm_memory_fence(VoidPtr virtual_address) { + if (!virtual_address || !hal_get_phys_address(virtual_address)) return kErrorInvalidData; + + asm volatile("clflush (%0)" : : "r"(virtual_address) : "memory"); + asm volatile("mfence" ::: "memory"); + + return kErrorSuccess; +} + /***********************************************************************************/ /// @brief Maps or allocates a page from virtual_address. /// @param virtual_address a valid virtual address. @@ -136,9 +148,7 @@ EXTERN_C Int32 mm_map_page(VoidPtr virtual_address, VoidPtr physical_address, UI hal_invl_tlb(virtual_address); - asm volatile("clflush (%0)" : : "r"(virtual_address) : "memory"); - - asm volatile("mfence" ::: "memory"); + mm_memory_fence(virtual_address); mmi_page_status(pte); diff --git a/dev/kernel/HALKit/AMD64/Processor.h b/dev/kernel/HALKit/AMD64/Processor.h index 1319277f..b57e9abf 100644 --- a/dev/kernel/HALKit/AMD64/Processor.h +++ b/dev/kernel/HALKit/AMD64/Processor.h @@ -272,6 +272,8 @@ EXTERN_C Void rt_cli(); EXTERN_C Void rt_sti(); EXTERN_C Void rt_cld(); EXTERN_C Void rt_std(); + +EXTERN_C Int32 mm_memory_fence(VoidPtr virtual_address); } // namespace Kernel::HAL EXTERN_C Kernel::Void idt_handle_generic(Kernel::UIntPtr rsp); diff --git a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc index 06c7781c..050826e3 100644 --- a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc +++ b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc @@ -257,7 +257,7 @@ SizeT drv_get_sector_count_ahci() { /// @brief Get the drive size. /// @return Disk size in bytes. SizeT drv_get_size_ahci() { - return drv_get_sector_count() * kAHCISectorSize; + return drv_std_get_sector_count() * kAHCISectorSize; } /// @brief Enable Host and probe using the IDENTIFY command. @@ -426,16 +426,16 @@ Bool drv_std_detected_ahci() { /// //////////////////////////////////////////////////// Void drv_std_write(UInt64 lba, Char* buffer, SizeT sector_sz, SizeT size_buffer) { - drv_std_input_output_ahci(lba / sector_sz, reinterpret_cast(buffer), sector_sz, - size_buffer); + drv_std_input_output_ahci(lba / sector_sz, reinterpret_cast(buffer), + sector_sz, size_buffer); } //////////////////////////////////////////////////// /// //////////////////////////////////////////////////// Void drv_std_read(UInt64 lba, Char* buffer, SizeT sector_sz, SizeT size_buffer) { - drv_std_input_output_ahci(lba / sector_sz, reinterpret_cast(buffer), sector_sz, - size_buffer); + drv_std_input_output_ahci(lba / sector_sz, reinterpret_cast(buffer), + sector_sz, size_buffer); } //////////////////////////////////////////////////// @@ -459,7 +459,7 @@ Bool drv_std_detected(Void) { @return Sector size in bytes. */ //////////////////////////////////////////////////// -SizeT drv_get_sector_count() { +SizeT drv_std_get_sector_count() { return drv_get_sector_count_ahci(); } @@ -467,7 +467,7 @@ SizeT drv_get_sector_count() { /// @brief Get the drive size. /// @return Disk size in bytes. //////////////////////////////////////////////////// -SizeT drv_get_size() { +SizeT drv_std_get_size() { return drv_get_size_ahci(); } diff --git a/dev/kernel/HALKit/AMD64/Storage/DMA+Generic.cc b/dev/kernel/HALKit/AMD64/Storage/DMA+Generic.cc index ca3e9ee4..4688203f 100644 --- a/dev/kernel/HALKit/AMD64/Storage/DMA+Generic.cc +++ b/dev/kernel/HALKit/AMD64/Storage/DMA+Generic.cc @@ -183,15 +183,15 @@ Boolean drv_std_detected(Void) { @return Number of sectors, or zero. */ /***********************************************************************************/ -Kernel::SizeT drv_get_sector_count() { +Kernel::SizeT drv_std_get_sector_count() { return (kATAIdentifyData[61] << 16) | kATAIdentifyData[60]; } /***********************************************************************************/ /// @brief Get the size of the current drive. /***********************************************************************************/ -Kernel::SizeT drv_get_size() { - return (drv_get_sector_count()) * kATASectorSize; +Kernel::SizeT drv_std_get_size() { + return (drv_std_get_sector_count()) * kATASectorSize; } #endif /* ifdef __ATA_DMA__ */ diff --git a/dev/kernel/HALKit/AMD64/Storage/PIO+Generic.cc b/dev/kernel/HALKit/AMD64/Storage/PIO+Generic.cc index a024d2fe..e57494a7 100644 --- a/dev/kernel/HALKit/AMD64/Storage/PIO+Generic.cc +++ b/dev/kernel/HALKit/AMD64/Storage/PIO+Generic.cc @@ -245,11 +245,11 @@ Void drv_std_write(UInt64 Lba, UInt16 IO, UInt8 Master, Char* Buf, SizeT SectorS drv_pio_std_write(Lba, IO, Master, Buf, SectorSz, Size); } -SizeT drv_get_size() { +SizeT drv_std_get_size() { return drv_pio_get_size(); } -SizeT drv_get_sector_count() { +SizeT drv_std_get_sector_count() { return drv_pio_get_sector_count(); } diff --git a/dev/kernel/KernelKit/FileMgr.h b/dev/kernel/KernelKit/FileMgr.h index 3dff61b0..2c4b2055 100644 --- a/dev/kernel/KernelKit/FileMgr.h +++ b/dev/kernel/KernelKit/FileMgr.h @@ -59,7 +59,7 @@ namespace Kernel { enum { - kFileIOInvalid = 0, + kFileIOInvalid = 0, kFileWriteAll = 100, kFileReadAll = 101, kFileReadChunk = 102, diff --git a/dev/kernel/KernelKit/KPC.h b/dev/kernel/KernelKit/KPC.h index e195d5ad..9de1f70f 100644 --- a/dev/kernel/KernelKit/KPC.h +++ b/dev/kernel/KernelKit/KPC.h @@ -61,7 +61,11 @@ inline constexpr KPCError kErrorInvalidCreds = 61; inline constexpr KPCError kErrorCDTrayBroken = 62; inline constexpr KPCError kErrorUnrecoverableDisk = 63; inline constexpr KPCError kErrorFileLocked = 64; -inline constexpr KPCError kErrorUnimplemented = -1; +inline constexpr KPCError kErrorDiskIsTooTiny = 65; +/// Kernel errors. +inline constexpr KPCError kErrorDmaExhausted = 101; +/// Generic errors. +inline constexpr KPCError kErrorUnimplemented = -1; /// @brief Does a system wide bug check. /// @param void no params are needed. diff --git a/dev/kernel/NewKit/KString.h b/dev/kernel/NewKit/KString.h index a9a58b7e..08f783c0 100644 --- a/dev/kernel/NewKit/KString.h +++ b/dev/kernel/NewKit/KString.h @@ -71,7 +71,7 @@ class KString final { }; class KStringBuilder final { -public: + public: static ErrorOr Construct(const Char* data); static const Char* FromBool(const Char* fmt, bool n); static const Char* Format(const Char* fmt, const Char* from); diff --git a/dev/kernel/StorageKit/DmaPool.h b/dev/kernel/StorageKit/DmaPool.h index 9e59910a..09851a0c 100644 --- a/dev/kernel/StorageKit/DmaPool.h +++ b/dev/kernel/StorageKit/DmaPool.h @@ -23,6 +23,7 @@ #pragma once #include +#include "NewKit/Defines.h" #ifdef __NE_AMD64__ #define NE_DMA_POOL_START (0x1000000) @@ -30,11 +31,14 @@ namespace Kernel { /// @brief DMA pool base pointer, here we're sure that AHCI or whatever tricky standard sees it. -inline UInt8* kDmaPoolPtr = (UInt8*) NE_DMA_POOL_START; +inline UInt8* kDmaPoolPtr = (UInt8*) NE_DMA_POOL_START; +inline const UInt8* kDmaPoolEnd = (UInt8*) (NE_DMA_POOL_START + NE_DMA_POOL_SIZE); +/***********************************************************************************/ /// @brief allocate from the rtl_dma_alloc system. /// @param size the size of the chunk to allocate. /// @param align alignement of pointer. +/***********************************************************************************/ inline VoidPtr rtl_dma_alloc(SizeT size, SizeT align) { if (!size) { return nullptr; @@ -42,11 +46,14 @@ inline VoidPtr rtl_dma_alloc(SizeT size, SizeT align) { UIntPtr addr = (UIntPtr) kDmaPoolPtr; - /// here we just align the address according to a `align` variable, i'd rather be a power of two really. + /// here we just align the address according to a `align` variable, i'd rather be a power of two + /// really. addr = (addr + (align - 1)) & ~(align - 1); - if ((addr + size) >= (NE_DMA_POOL_START + NE_DMA_POOL_SIZE)) { - kout << "DMA Pool exhausted!\r"; + if ((addr + size) >= reinterpret_cast(kDmaPoolEnd)) { + kout << "DMA Pool is exhausted!\r"; + + err_global_get() = kErrorDmaExhausted; return nullptr; } @@ -55,12 +62,18 @@ inline VoidPtr rtl_dma_alloc(SizeT size, SizeT align) { return (VoidPtr) addr; } +/***********************************************************************************/ +/// @brief Free DMA pointer. +/***********************************************************************************/ inline Void rtl_dma_free(SizeT size) { if (!size) return; kDmaPoolPtr = (UInt8*) (kDmaPoolPtr - size); } +/***********************************************************************************/ +/// @brief Flush DMA pointer. +/***********************************************************************************/ inline Void rtl_dma_flush(VoidPtr ptr, SizeT size_buffer) { if (ptr > (Void*) (NE_DMA_POOL_START + NE_DMA_POOL_SIZE)) { return; diff --git a/dev/kernel/src/DriveMgr.cc b/dev/kernel/src/DriveMgr.cc index 46a5c588..b0f78ae4 100644 --- a/dev/kernel/src/DriveMgr.cc +++ b/dev/kernel/src/DriveMgr.cc @@ -193,7 +193,7 @@ namespace Detail { trait.fKind = kMassStorageDrive | kUnformattedDrive | kReadOnlyDrive; trait.fSectorSz = 512; - trait.fLbaEnd = drv_get_sector_count() - 1; + trait.fLbaEnd = drv_std_get_sector_count() - 1; trait.fLbaStart = 0x400; } } diff --git a/dev/kernel/src/FS/HeFS+FileMgr.cc b/dev/kernel/src/FS/HeFS+FileMgr.cc index d7530d8f..e0b92a8d 100644 --- a/dev/kernel/src/FS/HeFS+FileMgr.cc +++ b/dev/kernel/src/FS/HeFS+FileMgr.cc @@ -10,5 +10,5 @@ #include #include -#endif // ifdef __FSKIT_INCLUDES_HEFS__ -#endif // ifndef __NE_MINIMAL_OS__ +#endif // ifdef __FSKIT_INCLUDES_HEFS__ +#endif // ifndef __NE_MINIMAL_OS__ diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 37c2f8fb..f4776686 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -756,7 +756,7 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input NE_UNUSED(flags); NE_UNUSED(part_name); - // verify disk. + // Verify Disk. drive->fVerify(drive->fPacket); // if disk isn't good, then error out. @@ -765,6 +765,12 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input return false; } + if (drv_std_get_size() < kHeFSMinimumDiskSize) { + err_global_get() = kErrorDiskIsTooTiny; + kout << "Error: Failed to allocate memory for boot node.\r"; + return NO; + } + HEFS_BOOT_NODE* root = new HEFS_BOOT_NODE(); if (!root) { @@ -806,11 +812,11 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input rt_string_len("fs/hefs-packet")); urt_copy_memory((VoidPtr) part_name, root->fVolName, urt_string_len(part_name)); - rt_copy_memory((VoidPtr) kHeFSMagic, root->fMagic, sizeof(kHeFSMagic)); + rt_copy_memory((VoidPtr) kHeFSMagic, root->fMagic, kHeFSMagicLen - 1); root->fBadSectors = 0; - root->fSectorCount = drv_get_sector_count(); + root->fSectorCount = drv_std_get_sector_count(); root->fSectorSize = drive->fSectorSz; @@ -821,7 +827,7 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input root->fINDCount = 0; - root->fDiskSize = drv_get_size(); + root->fDiskSize = drv_std_get_size(); root->fDiskStatus = kHeFSStatusUnlocked; root->fDiskFlags = flags; @@ -844,6 +850,8 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input root->fVID = kHeFSInvalidVID; + root->fChecksum = ke_calculate_crc32((Char*) root, sizeof(HEFS_BOOT_NODE)); + drive->fPacket.fPacketLba = start; drive->fPacket.fPacketSize = sizeof(HEFS_BOOT_NODE); drive->fPacket.fPacketContent = root; @@ -861,52 +869,55 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input start = root->fStartIND; - constexpr SizeT kHeFSPreallocateCount = 16UL; + constexpr SizeT kHeFSPreallocateCount = 0x10UL; - HEFS_INDEX_NODE_DIRECTORY* index_node = new HEFS_INDEX_NODE_DIRECTORY(); + HEFS_INDEX_NODE_DIRECTORY* dir = new HEFS_INDEX_NODE_DIRECTORY(); // Pre-allocate index node directory tree for (SizeT i = 0; i < kHeFSPreallocateCount; ++i) { - rt_set_memory(index_node, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); - urt_copy_memory((VoidPtr) u8"?", index_node->fName, urt_string_len(u8"?")); + rt_set_memory(dir, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); + urt_copy_memory((VoidPtr) u8".deleted", dir->fName, urt_string_len(u8".deleted")); + + dir->fFlags = flags; + dir->fKind = kHeFSFileKindDirectory; - index_node->fFlags = flags; - index_node->fKind = kHeFSFileKindDirectory; + dir->fDeleted = kHeFSTimeMax; /// TODO: Add current time. - index_node->fDeleted = kHeFSTimeMax; + dir->fEntryCount = 0; - index_node->fEntryCount = 1; + dir->fIndexNodeChecksum = 0; - index_node->fChecksum = 0; - index_node->fIndexNodeChecksum = 0; + dir->fUID = 0; + dir->fGID = 0; + dir->fMode = 0; - index_node->fUID = 0; - index_node->fGID = 0; - index_node->fMode = 0; + dir->fColor = kHeFSBlack; + dir->fChild = 0; + dir->fParent = 0; + dir->fNext = 0; + dir->fPrev = 0; - index_node->fColor = kHeFSBlack; - index_node->fChild = 0; - index_node->fParent = 0; - index_node->fNext = 0; - index_node->fPrev = 0; + dir->fChecksum = ke_calculate_crc32((Char*) dir, sizeof(HEFS_INDEX_NODE_DIRECTORY)); drive->fPacket.fPacketLba = start; drive->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); - drive->fPacket.fPacketContent = index_node; + drive->fPacket.fPacketContent = dir; start += sizeof(HEFS_INDEX_NODE_DIRECTORY); drive->fOutput(drive->fPacket); } - delete index_node; - index_node = nullptr; + delete dir; + dir = nullptr; delete root; root = nullptr; Detail::hefsi_balance_filesystem(root, drive); + err_global_get() = kErrorSuccess; + if (drive->fPacket.fPacketGood) return YES; err_global_get() = kErrorDiskIsCorrupted; @@ -953,7 +964,7 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu Detail::hefsi_balance_filesystem(root, drive); - auto dirent = new HEFS_INDEX_NODE_DIRECTORY(); + HEFS_INDEX_NODE_DIRECTORY* dirent = new HEFS_INDEX_NODE_DIRECTORY(); rt_set_memory(dirent, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); @@ -969,7 +980,7 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu dirent->fFlags = flags; dirent->fChecksum = 0; - dirent->fEntryCount = 1; + dirent->fEntryCount = 0; if (Detail::hefs_allocate_index_directory_node(root, drive, dirent)) { delete dirent; @@ -1093,7 +1104,7 @@ Boolean fs_init_hefs(Void) noexcept { HeFileSystemParser parser; - parser.Format(&drv, kHeFSEncodingUTF16, kHeFSDefaultVoluneName); + parser.Format(&drv, kHeFSEncodingUTF8, kHeFSDefaultVoluneName); return YES; } diff --git a/dev/kernel/src/FS/NeFS.cc b/dev/kernel/src/FS/NeFS.cc index e68a4035..d572bffc 100644 --- a/dev/kernel/src/FS/NeFS.cc +++ b/dev/kernel/src/FS/NeFS.cc @@ -32,12 +32,12 @@ using namespace Kernel; /***********************************************************************************/ /// @brief get sector count. /***********************************************************************************/ -Kernel::SizeT drv_get_sector_count(); +Kernel::SizeT drv_std_get_sector_count(); /***********************************************************************************/ /// @brief get device size. /***********************************************************************************/ -Kernel::SizeT drv_get_size(); +Kernel::SizeT drv_std_get_size(); #endif @@ -455,8 +455,8 @@ bool NeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input const I rt_copy_memory((VoidPtr) kNeFSUntitledHD, (VoidPtr) part_block->PartitionName, rt_string_len(kNeFSUntitledHD)); - SizeT sectorCount = drv_get_sector_count(); - SizeT diskSize = drv_get_size(); + SizeT sectorCount = drv_std_get_sector_count(); + SizeT diskSize = drv_std_get_size(); part_block->Version = kNeFSVersionInteger; diff --git a/dev/kernel/src/Utils.cc b/dev/kernel/src/Utils.cc index ea0ff7a7..4f47849b 100644 --- a/dev/kernel/src/Utils.cc +++ b/dev/kernel/src/Utils.cc @@ -193,4 +193,3 @@ EXTERN_C void* memcpy(void* dst, const void* src, long long unsigned int len) { EXTERN_C Kernel::Int32 strcmp(const char* dst, const char* src) { return Kernel::rt_string_cmp(dst, src, Kernel::rt_string_len(dst)); } - diff --git a/dev/modules/AHCI/AHCI.h b/dev/modules/AHCI/AHCI.h index 98fb3d60..7303e8c5 100644 --- a/dev/modules/AHCI/AHCI.h +++ b/dev/modules/AHCI/AHCI.h @@ -349,10 +349,10 @@ Kernel::Void drv_std_write(Kernel::UInt64 lba, Kernel::Char* buf, Kernel::SizeT Kernel::SizeT buf_sz); /// @brief Gets the sector count from AHCI disk. -Kernel::SizeT drv_get_sector_count(); +Kernel::SizeT drv_std_get_sector_count(); /// @brief Gets the AHCI disk size. -Kernel::SizeT drv_get_size(); +Kernel::SizeT drv_std_get_size(); /// @brief Checks if the drive has completed the command. Kernel::Bool drv_is_ready(void); diff --git a/dev/modules/ATA/ATA.h b/dev/modules/ATA/ATA.h index bd21d106..e7cf200f 100644 --- a/dev/modules/ATA/ATA.h +++ b/dev/modules/ATA/ATA.h @@ -149,9 +149,9 @@ Kernel::Void drv_std_write(Kernel::UInt64 lba, Kernel::UInt16 io, Kernel::UInt8 Kernel::Char* buf, Kernel::SizeT sec_sz, Kernel::SizeT buf_sz); /// @brief get sector count. -Kernel::SizeT drv_get_sector_count(); +Kernel::SizeT drv_std_get_sector_count(); /// @brief get device size. -Kernel::SizeT drv_get_size(); +Kernel::SizeT drv_std_get_size(); #endif // ifdef __NEOSKRNL__ \ No newline at end of file diff --git a/dev/user/ProcessCodes.h b/dev/user/ProcessCodes.h index 74b50c68..90457944 100644 --- a/dev/user/ProcessCodes.h +++ b/dev/user/ProcessCodes.h @@ -51,6 +51,7 @@ inline constexpr ErrRef kErrorInvalidCreds = 61; inline constexpr ErrRef kErrorCDTrayBroken = 62; inline constexpr ErrRef kErrorUnrecoverableDisk = 63; inline constexpr ErrRef kErrorFileLocked = 64; +inline constexpr ErrRef kErrorDiskIsTooTiny = 65; inline constexpr ErrRef kErrorUnimplemented = -1; /// @brief The last error reported by the system to the process. diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index 75f98bcc..f3c2d6bf 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -26,7 +26,7 @@ \title{HeFS (High-Throughput Extended File System) Specification} \author{Amlal El Mahrouss} -\date{2024--2025} +\date{2024-2025} \begin{document} diff --git a/tooling/hefs.h b/tooling/hefs.h index d0ad823b..2a04a1c3 100644 --- a/tooling/hefs.h +++ b/tooling/hefs.h @@ -1,6 +1,6 @@ /* ------------------------------------------- - Copyright (C) 2025, Amlal El Mahrouss, all rights reserved. + Copyright (C) 2024-2025, Amlal El Mahrouss, all rights reserved. ------------------------------------------- */ @@ -16,17 +16,11 @@ #define kHeFSFileNameLen (256U) #define kHeFSPartNameLen (128U) -#define kHeFSMinimumDiskSize (gib_cast(4)) - -#define kHeFSDefaultVoluneName u"HeFS Volume" - -#define kHeFSDIMBootDir u"boot-x/dir" -#define kHeFSMIMEBootFile u"boot-x/file" - -#define kHeFSDIMSystemDir u"system-x/dir" -#define kHeFSMIMESystemFile u"system-x/file" +#define kHeFSDefaultVoluneName u8"HeFS Volume" namespace mkfs::hefs { + +// Drive kinds enum { kHeFSHardDrive = 0xC0, // Hard Drive kHeFSSolidStateDrive = 0xC1, // Solid State Drive @@ -38,6 +32,7 @@ enum { kHeFSDriveCount = 7, }; +// Disk status enum { kHeFSStatusUnlocked = 0x18, kHeFSStatusLocked, @@ -46,6 +41,7 @@ enum { kHeFSStatusCount, }; +// Encodings enum { kHeFSEncodingUTF8 = 0x00, kHeFSEncodingUTF16, @@ -60,18 +56,13 @@ enum { kHeFSEncodingCount, }; -// Constants +// Block constants constexpr std::size_t kHeFSBlockCount = 16; -// Types +// Time type using ATime = std::uint64_t; -enum { - kHeFSRed = 100, - kHeFSBlack, - kHeFSColorCount, -}; - +// File kinds inline constexpr uint16_t kHeFSFileKindRegular = 0x00; inline constexpr uint16_t kHeFSFileKindDirectory = 0x01; inline constexpr uint16_t kHeFSFileKindBlock = 0x02; @@ -82,14 +73,22 @@ inline constexpr uint16_t kHeFSFileKindSymbolicLink = 0x06; inline constexpr uint16_t kHeFSFileKindUnknown = 0x07; inline constexpr uint16_t kHeFSFileKindCount = 0x08; -// Basic Time Constants +// Red-black tree colors +enum { + kHeFSInvalidColor = 0, + kHeFSRed = 100, + kHeFSBlack, + kHeFSColorCount, +}; + +// Time constants inline constexpr ATime kHeFSTimeInvalid = 0x0000000000000000; inline constexpr ATime kHeFSTimeMax = 0xFFFFFFFFFFFFFFFF - 1; -// Boot Node (Superblock Equivalent) +// Boot Node struct alignas(8) BootNode { char magic[kHeFSMagicLen]{}; - char16_t volumeName[kHeFSPartNameLen]{}; + char8_t volumeName[kHeFSPartNameLen]{}; std::uint32_t version{}; std::uint64_t badSectors{}; std::uint64_t sectorCount{}; @@ -108,19 +107,16 @@ struct alignas(8) BootNode { std::uint64_t reserved2{}; std::uint64_t reserved3{}; std::uint64_t reserved4{}; + char pad[272]{}; }; -// File Node (Index Node) +// Index Node struct alignas(8) IndexNode { - char16_t name[kHeFSFileNameLen]{}; + char8_t name[kHeFSFileNameLen]{}; std::uint32_t flags{}; std::uint16_t kind{}; std::uint32_t size{}; std::uint32_t checksum{}; - std::uint32_t recoverChecksum{}; - std::uint32_t blockChecksum{}; - std::uint32_t linkChecksum{}; - char16_t mime[kHeFSFileNameLen]{}; bool symbolicLink{false}; ATime created{}; ATime accessed{}; @@ -129,23 +125,18 @@ struct alignas(8) IndexNode { std::uint32_t uid{}; std::uint32_t gid{}; std::uint32_t mode{}; - std::uint64_t blockLinkStart[kHeFSBlockCount]{}; - std::uint64_t blockLinkEnd[kHeFSBlockCount]{}; - std::uint64_t blockStart[kHeFSBlockCount]{}; - std::uint64_t blockEnd[kHeFSBlockCount]{}; - std::uint64_t blockRecoveryStart[kHeFSBlockCount]{}; - std::uint64_t blockRecoveryEnd[kHeFSBlockCount]{}; + std::uint64_t block[kHeFSBlockCount]{}; + char pad[62]{}; }; -// Directory Node (Red-Black Tree Node) +// Index Node Directory (Red-Black Tree Node) struct alignas(8) IndexNodeDirectory { - char16_t name[kHeFSFileNameLen]{}; + char8_t name[kHeFSFileNameLen]{}; std::uint32_t flags{}; std::uint16_t kind{}; std::uint32_t entryCount{}; std::uint32_t checksum{}; std::uint32_t indexNodeChecksum{}; - char16_t dim[kHeFSFileNameLen]{}; ATime created{}; ATime accessed{}; ATime modified{}; @@ -153,13 +144,13 @@ struct alignas(8) IndexNodeDirectory { std::uint32_t uid{}; std::uint32_t gid{}; std::uint32_t mode{}; - std::uint64_t indexNodeStart[kHeFSBlockCount]{}; - std::uint64_t indexNodeEnd[kHeFSBlockCount]{}; + std::uint64_t indexNode[kHeFSBlockCount]{}; std::uint8_t color{}; std::uint64_t next{}; std::uint64_t prev{}; std::uint64_t child{}; std::uint64_t parent{}; + char pad[32]{}; }; } // namespace mkfs::hefs diff --git a/tooling/mkfs.hefs.cc b/tooling/mkfs.hefs.cc index 542ae175..3147a6e6 100644 --- a/tooling/mkfs.hefs.cc +++ b/tooling/mkfs.hefs.cc @@ -28,10 +28,10 @@ static std::basic_string get_option(const std::basic_string& } } // namespace mkfs::detail -static size_t kDiskSize = 1024 * 1024 * 1024 * 4UL; -static uint16_t kVersion = kHeFSVersion; -static std::u16string kLabel = kHeFSDefaultVoluneName; -static size_t kSectorSize = 512; +static size_t kDiskSize = 1024 * 1024 * 1024 * 4UL; +static uint16_t kVersion = kHeFSVersion; +static std::u8string kLabel = kHeFSDefaultVoluneName; +static size_t kSectorSize = 512; int main(int argc, char** argv) { if (argc < 2) { @@ -42,8 +42,8 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - std::string args; - std::u16string args_wide; + std::string args; + std::u8string args_wide; for (int i = 1; i < argc; ++i) { args += argv[i]; @@ -55,13 +55,13 @@ int main(int argc, char** argv) { args_wide.push_back(ch); } - args_wide += u" "; + args_wide += u8" "; } auto output_device = mkfs::detail::get_option(args, "-o"); kSectorSize = std::strtol(mkfs::detail::get_option(args, "-s").data(), nullptr, 10); - kLabel = mkfs::detail::get_option(args_wide, u"-L"); + kLabel = mkfs::detail::get_option(args_wide, u8"-L"); if (kLabel.empty()) kLabel = kHeFSDefaultVoluneName; @@ -91,14 +91,14 @@ int main(int argc, char** argv) { bootNode.version = kVersion; bootNode.diskKind = mkfs::hefs::kHeFSHardDrive; - bootNode.encoding = mkfs::hefs::kHeFSEncodingUTF16; + bootNode.encoding = mkfs::hefs::kHeFSEncodingUTF8; bootNode.diskSize = kDiskSize; bootNode.sectorSize = kSectorSize; bootNode.startIND = start_ind; bootNode.endIND = end_ind; bootNode.diskStatus = mkfs::hefs::kHeFSStatusUnlocked; - std::memcpy(bootNode.magic, kHeFSMagic, kHeFSMagicLen); + std::memcpy(bootNode.magic, kHeFSMagic, kHeFSMagicLen - 1); std::memcpy(bootNode.volumeName, kLabel.data(), kLabel.size() * sizeof(char16_t)); filesystem.seekp(std::strtol(mkfs::detail::get_option(args, "-p").data(), nullptr, 10)); @@ -121,9 +121,9 @@ int main(int argc, char** argv) { for (size_t i = 0; i < cnt; ++i) { mkfs::hefs::IndexNodeDirectory indexNode{}; - std::memcpy(indexNode.name, u"/", std::u16string(u"/").size() * sizeof(char16_t)); + std::memcpy(indexNode.name, u8"/", std::u8string(u8"/").size() * sizeof(char16_t)); - indexNode.flags = mkfs::hefs::kHeFSEncodingUTF16; + indexNode.flags = mkfs::hefs::kHeFSEncodingUTF8; indexNode.kind = mkfs::hefs::kHeFSFileKindDirectory; indexNode.deleted = mkfs::hefs::kHeFSTimeMax; -- cgit v1.2.3 From 9610712ae4a793f50c30e8a45676f830cba09588 Mon Sep 17 00:00:00 2001 From: Amlal Date: Mon, 28 Apr 2025 19:32:21 +0200 Subject: dev, kernel: Fixed HeFS directory allocation routine. Signed-off-by: Amlal --- dev/kernel/FSKit/HeFS.h | 12 +- dev/kernel/FSKit/NeFS.h | 4 +- dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc | 16 ++- dev/kernel/src/FS/HeFS.cc | 173 ++++++++++-------------- docs/tex/hefs.tex | 2 +- 5 files changed, 94 insertions(+), 113 deletions(-) (limited to 'docs') diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index a6fab1c5..feaee7bc 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -24,7 +24,7 @@ #define kHeFSFileNameLen (256U) #define kHeFSPartNameLen (128U) -#define kHeFSMinimumDiskSize (gib_cast(4)) +#define kHeFSMinimumDiskSize (mib_cast(16)) #define kHeFSDefaultVoluneName u8"HeFS Volume" @@ -108,7 +108,7 @@ struct PACKED HEFS_BOOT_NODE final { /// Drive, etc). Kernel::UInt8 fEncoding; /// @brief Encoding of the filesystem. (UTF-8, UTF-16, etc). Kernel::UInt64 fStartIND; /// @brief Start of the INode tree. - Kernel::UInt64 fEndIND; /// @brief End of the INode tree. + Kernel::UInt64 fEndIND; /// @brief End of the INode tree. it is used to track down the last ind offset. Kernel::UInt64 fINDCount; /// @brief Number of leafs in the INode tree. Kernel::UInt64 fDiskSize; /// @brief Size of the disk. (Could be a virtual size, that is not the /// real size of the disk.) @@ -116,10 +116,10 @@ struct PACKED HEFS_BOOT_NODE final { Kernel::UInt16 fDiskFlags; /// @brief Flags of the disk. (read-only, read-write, etc). Kernel::UInt16 fVID; /// @brief Virtual Identification Number within an EPM disk. (0xFFFF if not used). - Kernel::UInt64 fReserved; /// @brief Reserved for future use. - Kernel::UInt64 fReserved2; /// @brief Reserved for future use. - Kernel::UInt64 fReserved3; /// @brief Reserved for future use. - Kernel::UInt64 fReserved4; /// @brief Reserved for future use. + Kernel::UInt64 fStartIN; /// @brief Reserved for future use. + Kernel::UInt64 fEndIN; /// @brief Reserved for future use. + Kernel::UInt64 fReserved; /// @brief Reserved for future use. + Kernel::UInt64 fReserved1; /// @brief Reserved for future use. Kernel::Char fPad[272]; }; diff --git a/dev/kernel/FSKit/NeFS.h b/dev/kernel/FSKit/NeFS.h index 2235ca4f..588cbe69 100644 --- a/dev/kernel/FSKit/NeFS.h +++ b/dev/kernel/FSKit/NeFS.h @@ -108,7 +108,7 @@ default. #define kNeFSFlagCreated (71) #define kNeFSMimeNameLen (200) -#define kNeFSForkNameLen (200) +#define kNeFSForkNameLen (199) #define kNeFSFrameworkExt ".fwrk/" #define kNeFSStepsExt ".step/" @@ -200,8 +200,6 @@ struct PACKED NEFS_FORK_STRUCT final { Kernel::Lba NextSibling; Kernel::Lba PreviousSibling; - - Kernel::Char Pad[2] = {0}; }; /// @brief Partition block type diff --git a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc index c3c49533..3994a1a9 100644 --- a/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc +++ b/dev/kernel/HALKit/AMD64/Storage/AHCI+Generic.cc @@ -248,6 +248,14 @@ STATIC Void drv_std_input_output_ahci(UInt64 lba, UInt8* buffer, SizeT sector_sz rtl_dma_flush(ptr, size_buffer); } + if ((kSATAHba->Ports[kSATAIndex].Tfd & (kSATASRBsy | kSATASRDrq)) == 0) { + goto ahci_io_end; + } else { + kout << "Warning: Disk still busy after command completion!\r"; + while (kSATAHba->Ports[kSATAIndex].Tfd & (kSATASRBsy | kSATASRDrq)); + } + + ahci_io_end: rtl_dma_free(size_buffer); err_global_get() = kErrorSuccess; @@ -434,16 +442,16 @@ Bool drv_std_detected_ahci() { /// //////////////////////////////////////////////////// Void drv_std_write(UInt64 lba, Char* buffer, SizeT sector_sz, SizeT size_buffer) { - drv_std_input_output_ahci(lba, reinterpret_cast(buffer), - sector_sz, size_buffer); + drv_std_input_output_ahci(lba, reinterpret_cast(buffer), sector_sz, + size_buffer); } //////////////////////////////////////////////////// /// //////////////////////////////////////////////////// Void drv_std_read(UInt64 lba, Char* buffer, SizeT sector_sz, SizeT size_buffer) { - drv_std_input_output_ahci(lba, reinterpret_cast(buffer), - sector_sz, size_buffer); + drv_std_input_output_ahci(lba, reinterpret_cast(buffer), sector_sz, + size_buffer); } //////////////////////////////////////////////////// diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 4f05214a..94f7dac1 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -93,6 +93,8 @@ namespace Detail { } else { start += sizeof(HEFS_INDEX_NODE_DIRECTORY); } + + kout << "Traversing RB-Tree...\r"; } /***********************************************************************************/ @@ -152,7 +154,7 @@ namespace Detail { delete grand_parent; grand_parent = nullptr; - dir->fColor = kHeFSBlack; + kout << "Rotate tree to left.\r"; } /***********************************************************************************/ @@ -196,6 +198,8 @@ namespace Detail { mnt->fPacket.fPacketContent = dir; mnt->fOutput(mnt->fPacket); + + kout << "Rotate tree to right.\r"; } /// @brief Get the index node size. @@ -280,7 +284,7 @@ namespace Detail { } STATIC _Output BOOL hefs_allocate_index_directory_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, - HEFS_INDEX_NODE_DIRECTORY* dir) noexcept { + const Utf8Char* dir_name) noexcept { if (root && mnt) { HEFS_INDEX_NODE_DIRECTORY* tmpdir = new HEFS_INDEX_NODE_DIRECTORY(); @@ -299,32 +303,60 @@ namespace Detail { mnt->fInput(mnt->fPacket); if (!mnt->fPacket.fPacketGood) { - delete tmpdir; - return NO; + break; } - if (!tmpdir->fCreated && tmpdir->fDeleted) { + if ((!tmpdir->fCreated && tmpdir->fDeleted) || *tmpdir->fName == u8'\0') { + HEFS_INDEX_NODE_DIRECTORY* dirent = new HEFS_INDEX_NODE_DIRECTORY(); + + rt_set_memory(dirent, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); + + urt_copy_memory((VoidPtr) dir_name, dirent->fName, urt_string_len(dir_name)); + + dirent->fAccessed = 0; + dirent->fCreated = 1UL; /// TODO: Add the current time. + dirent->fDeleted = 0; + dirent->fModified = 0; + dirent->fEntryCount = 0; + + dirent->fKind = kHeFSFileKindDirectory; + dirent->fFlags = kHeFSEncodingUTF8; + dirent->fChecksum = 0; + + dirent->fChecksum = ke_calculate_crc32((Char*) dirent, sizeof(HEFS_INDEX_NODE_DIRECTORY)); + + dirent->fEntryCount = 0; + + dirent->fChild = tmpdir->fChild; + dirent->fParent = tmpdir->fParent; + dirent->fNext = tmpdir->fNext; + dirent->fPrev = tmpdir->fPrev; + dirent->fColor = tmpdir->fColor; + mnt->fPacket.fPacketLba = start; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); - mnt->fPacket.fPacketContent = dir; - - dir->fChild = tmpdir->fChild; - dir->fParent = tmpdir->fParent; - dir->fNext = tmpdir->fNext; - dir->fPrev = tmpdir->fPrev; - dir->fColor = tmpdir->fColor; + mnt->fPacket.fPacketContent = dirent; mnt->fOutput(mnt->fPacket); + delete dirent; + dirent = nullptr; + + delete tmpdir; + tmpdir = nullptr; + + err_global_get() = kErrorSuccess; + return YES; } - auto old_start = start; hefsi_traverse_tree(tmpdir, root, start); - if (start > root->fEndIND || start == old_start) break; } + err_global_get() = kErrorDisk; + delete tmpdir; + return NO; } err_global_get() = kErrorDiskIsFull; @@ -336,8 +368,8 @@ namespace Detail { if (root && mnt) { HEFS_INDEX_NODE_DIRECTORY* dir = new HEFS_INDEX_NODE_DIRECTORY(); - auto start = root->fStartIND; - auto end = root->fEndIND; + auto start = mnt->fLbaStart + kHeFSINDStartLBA; + auto end = mnt->fLbaEnd; auto hop_watch = 0UL; @@ -375,9 +407,8 @@ namespace Detail { } } - auto old_start = start; hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; } delete dir; @@ -487,9 +518,8 @@ namespace Detail { } } - auto old_start = start; hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; } delete dir; @@ -568,11 +598,9 @@ namespace Detail { } } - auto old_start = start; - hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; } delete dir; @@ -688,20 +716,18 @@ namespace Detail { return NO; } - auto old_start = start; hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; continue; } else { if (dir_parent->fNext == start) { hefsi_rotate_left(dir, start, mnt); - auto old_start = start; hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; continue; } @@ -726,10 +752,9 @@ namespace Detail { hefsi_rotate_right(dir, start, mnt); } - auto old_start = start; hefsi_traverse_tree(dir, root, start); - if (start > root->fEndIND || start == old_start) break; + if (start > root->fEndIND) break; } delete dir; @@ -821,7 +846,7 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input root->fSectorSize = drive->fSectorSz; - Lba start = drive->fLbaStart + kHeFSINDStartLBA; + SizeT start = drive->fLbaStart + kHeFSINDStartLBA; if (start > drive->fLbaEnd) { delete root; @@ -850,9 +875,7 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input } root->fReserved = 0; - root->fReserved2 = 0; - root->fReserved3 = 0; - root->fReserved4 = 0; + root->fReserved1 = 0; root->fVersion = kHeFSVersion; @@ -875,56 +898,21 @@ _Output Bool HeFileSystemParser::Format(_Input _Output DriveTrait* drive, _Input return NO; } - start = root->fStartIND; - - constexpr SizeT kHeFSPreallocateCount = 0x10UL; + constexpr SizeT kHeFSPreallocateCount = 0x7UL; - HEFS_INDEX_NODE_DIRECTORY* dir = new HEFS_INDEX_NODE_DIRECTORY(); + const Utf8Char* kFileMap[kHeFSPreallocateCount] = { + u8"/", u8"/boot", u8"/system", u8"/devices", u8"/network", u8"/users", u8"/home", + }; - // Pre-allocate index node directory tree for (SizeT i = 0; i < kHeFSPreallocateCount; ++i) { - rt_set_memory(dir, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); - urt_copy_memory((VoidPtr) u8".deleted", dir->fName, urt_string_len(u8".deleted")); - - dir->fFlags = flags; - dir->fKind = kHeFSFileKindDirectory; - - dir->fCreated = 0; - dir->fDeleted = kHeFSTimeMax; /// TODO: Add current time. - - dir->fEntryCount = 0; - - dir->fIndexNodeChecksum = 0; - - dir->fUID = 0; - dir->fGID = 0; - dir->fMode = 0; - - dir->fColor = kHeFSBlack; - dir->fChild = 0; - dir->fParent = 0; - dir->fNext = 0; - dir->fPrev = 0; - - dir->fChecksum = ke_calculate_crc32((Char*) dir, sizeof(HEFS_INDEX_NODE_DIRECTORY)); - - drive->fPacket.fPacketLba = start; - drive->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE_DIRECTORY); - drive->fPacket.fPacketContent = dir; - - start += sizeof(HEFS_INDEX_NODE_DIRECTORY); - - drive->fOutput(drive->fPacket); + this->CreateDirectory(drive, kHeFSEncodingUTF8, kFileMap[i]); } - delete dir; - dir = nullptr; + Detail::hefsi_balance_filesystem(root, drive); delete root; root = nullptr; - Detail::hefsi_balance_filesystem(root, drive); - err_global_get() = kErrorSuccess; if (drive->fPacket.fPacketGood) return YES; @@ -943,6 +931,8 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu const Utf8Char* dir) { HEFS_BOOT_NODE* root = new HEFS_BOOT_NODE(); + NE_UNUSED(flags); + kout << "CreateDirectory...\r"; if (!root) { @@ -964,41 +954,26 @@ _Output Bool HeFileSystemParser::CreateDirectory(_Input DriveTrait* drive, _Inpu drive->fInput(drive->fPacket); - HEFS_INDEX_NODE_DIRECTORY* dirent = new HEFS_INDEX_NODE_DIRECTORY(); - - rt_set_memory(dirent, 0, sizeof(HEFS_INDEX_NODE_DIRECTORY)); - - urt_copy_memory((VoidPtr) dir, dirent->fName, urt_string_len(dir)); - - dirent->fAccessed = 0; - dirent->fCreated = kHeFSTimeMax; /// TODO: Add the current time. - dirent->fDeleted = 0; - dirent->fModified = 0; - dirent->fEntryCount = 0; + if (!KStringBuilder::Equals(root->fMagic, kHeFSMagic) || root->fVersion != kHeFSVersion) { + delete root; + root = nullptr; - dirent->fKind = kHeFSFileKindDirectory; - dirent->fFlags = flags; - dirent->fChecksum = 0; + err_global_get() = kErrorDiskIsCorrupted; - dirent->fChecksum = ke_calculate_crc32((Char*) dirent, sizeof(HEFS_INDEX_NODE_DIRECTORY)); + kout << "Invalid BootNode!\r"; - dirent->fEntryCount = 0; + return NO; + } - if (Detail::hefs_allocate_index_directory_node(root, drive, dirent)) { - delete dirent; - dirent = nullptr; + if (Detail::hefs_allocate_index_directory_node(root, drive, dir)) { + Detail::hefsi_balance_filesystem(root, drive); delete root; root = nullptr; - Detail::hefsi_balance_filesystem(root, drive); - return YES; } - delete dirent; - dirent = nullptr; - delete root; root = nullptr; @@ -1068,7 +1043,7 @@ _Output Bool HeFileSystemParser::CreateFile(_Input DriveTrait* drive, _Input con } node->fAccessed = 0; - node->fCreated = kHeFSTimeMax; + node->fCreated = 1UL; node->fDeleted = 0; node->fModified = 0; node->fSize = 0; diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index f3c2d6bf..a88b2270 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -137,7 +137,7 @@ HeFS is designed with the following objectives: \section{Minimum Requirements} -HeFS expects a minimum disk size of 4GB. Optimal performance is achieved with 8GB or more. +HeFS expects a minimum disk size of 16MB. Optimal performance is aimed to be achieved with 8GB or more. Supports: HDDs, SSDs, USB mass storage, SCSI/SAS, optical drives. \section{Future Work} -- cgit v1.2.3 From 911bf2d68a4904d8ff838d31e00ef65d4e03063c Mon Sep 17 00:00:00 2001 From: Amlal Date: Wed, 30 Apr 2025 10:12:55 +0200 Subject: kernel, sched: worked on a pending ticket (process rebalancing) Signed-off-by: Amlal --- dev/ddk/DDKKit/ifs.h | 16 +++++++++++++++ dev/kernel/KernelKit/UserProcessScheduler.h | 3 ++- dev/kernel/src/UserProcessScheduler.cc | 32 ++++++++++++++++++++++++++--- docs/md/SPECIFICATION_KERNEL.md | 4 +++- 4 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 dev/ddk/DDKKit/ifs.h (limited to 'docs') diff --git a/dev/ddk/DDKKit/ifs.h b/dev/ddk/DDKKit/ifs.h new file mode 100644 index 00000000..0665de9a --- /dev/null +++ b/dev/ddk/DDKKit/ifs.h @@ -0,0 +1,16 @@ +/* ------------------------------------------- + + Copyright Amlal El Mahrouss. + + FILE: ifs.h + PURPOSE: IFS model base header. + +------------------------------------------- */ + +#pragma once + +#include + +struct DDK_IFS_MANIFEST; + +/// @brief IFS hooks to plug into the FileMgr. diff --git a/dev/kernel/KernelKit/UserProcessScheduler.h b/dev/kernel/KernelKit/UserProcessScheduler.h index d6cd7ad4..1e8719ca 100644 --- a/dev/kernel/KernelKit/UserProcessScheduler.h +++ b/dev/kernel/KernelKit/UserProcessScheduler.h @@ -106,7 +106,8 @@ class USER_PROCESS final { kExecutableKindCount, }; - ProcessTime PTime{0}; //! @brief USER_PROCESS allocated tine. + ProcessTime PTime{0}; //! @brief Process allocated tine. + ProcessTime RTime{0}; //! @brief Process Run time PID ProcessId{kSchedInvalidPID}; Int32 Kind{kExecutableKind}; diff --git a/dev/kernel/src/UserProcessScheduler.cc b/dev/kernel/src/UserProcessScheduler.cc index 96938d10..4b206317 100644 --- a/dev/kernel/src/UserProcessScheduler.cc +++ b/dev/kernel/src/UserProcessScheduler.cc @@ -20,6 +20,8 @@ #include #include #include +#include "KernelKit/CoreProcessScheduler.h" +#include "NewKit/Defines.h" ///! BUGS: 0 @@ -424,6 +426,7 @@ ProcessID UserProcessScheduler::Spawn(const Char* name, VoidPtr code, VoidPtr im process.ProcessId = pid; process.Status = ProcessStatusKind::kStarting; process.PTime = (UIntPtr) AffinityKind::kStandard; + process.RTime = 0; (Void)(kout << "PID: " << number(process.ProcessId) << kendl); (Void)(kout << "Name: " << process.Name << kendl); @@ -510,13 +513,36 @@ SizeT UserProcessScheduler::Run() noexcept { process.StackFrame, process.ProcessId); if (!ret) { - if (process.Affinity == AffinityKind::kRealTime) continue; - kout << "The process: " << process.Name << ", is not valid! Crashing it...\r"; - process.Crash(); } } else { + if (process.ProcessId == this->CurrentProcess().Leak().ProcessId) { + if (process.PTime < process.RTime) { + if (process.RTime < (Int32) AffinityKind::kRealTime) + process.PTime = (Int32) AffinityKind::kVeryLowUsage; + else if (process.RTime < (Int32) AffinityKind::kVeryHigh) + process.PTime = (Int32) AffinityKind::kLowUsage; + else if (process.RTime < (Int32) AffinityKind::kHigh) + process.PTime = (Int32) AffinityKind::kStandard; + else if (process.RTime < (Int32) AffinityKind::kStandard) + process.PTime = (Int32) AffinityKind::kHigh; + + process.RTime = static_cast(process.Affinity); + + BOOL ret = UserProcessHelper::Switch(process.Image.Leak().Leak().Leak(), + &process.StackReserve[process.StackSize - 1], + process.StackFrame, process.ProcessId); + + if (!ret) { + kout << "The process: " << process.Name << ", is not valid! Crashing it...\r"; + process.Crash(); + } + } else { + ++process.RTime; + } + } + --process.PTime; } } diff --git a/docs/md/SPECIFICATION_KERNEL.md b/docs/md/SPECIFICATION_KERNEL.md index ae283428..6f1fae03 100644 --- a/docs/md/SPECIFICATION_KERNEL.md +++ b/docs/md/SPECIFICATION_KERNEL.md @@ -24,7 +24,7 @@ - TLS (Thread Local Storage) support. - BinaryMutex, Locks, Timers. - Canary mechanisms. -- Dynamic Sys. +- Dynamic Loader. - Cross Platform. - Permission Selectors. @@ -51,6 +51,8 @@ - Hal -> hal_foo_bar - Class methods -> Class::FooBar +Internals function shall be formated as such: (hali, rtli, rti...) + =================================== # 4: The Bootloader (BootZ) -- cgit v1.2.3 From 72fa8a81498a53491950f92d9e96fbe6dd65cc04 Mon Sep 17 00:00:00 2001 From: Amlal Date: Thu, 1 May 2025 08:30:58 +0200 Subject: dev, kernel: HeFS spec and tooling, and PEF spec updates. Signed-off-by: Amlal --- dev/kernel/FSKit/HeFS.h | 6 +++--- dev/kernel/KernelKit/PEF.h | 8 +++++--- dev/kernel/src/FS/HeFS.cc | 8 ++++---- docs/tex/hefs.tex | 8 ++++---- tooling/hefs.h | 51 ++-------------------------------------------- tooling/mkfs.hefs.cc | 1 + 6 files changed, 19 insertions(+), 63 deletions(-) (limited to 'docs') diff --git a/dev/kernel/FSKit/HeFS.h b/dev/kernel/FSKit/HeFS.h index 80ef6646..da5da3c1 100644 --- a/dev/kernel/FSKit/HeFS.h +++ b/dev/kernel/FSKit/HeFS.h @@ -83,7 +83,7 @@ inline constexpr UInt16 kHeFSFileKindCount = 0x08; /// @brief HeFS blocks are array containing sparse blocks of data. /// @details The blocks are used to store the data of a file. Each block is a pointer to a block of /// data on the disk. -inline constexpr UInt16 kHeFSBlockCount = 0x10; +inline constexpr UInt16 kHeFSSliceCount = 0x10; inline constexpr UInt16 kHeFSInvalidVID = 0xFFFF; @@ -146,7 +146,7 @@ struct PACKED HEFS_INDEX_NODE final { Kernel::UInt32 fUID, fGID; /// @brief User ID and Group ID of the file. Kernel::UInt32 fMode; /// @brief File mode. (read, write, execute, etc). - Kernel::UInt64 fBlock[kHeFSBlockCount]; /// @brief block slice. + Kernel::UInt64 fBlock[kHeFSSliceCount]; /// @brief block slice. Kernel::Char fPad[69]; }; @@ -180,7 +180,7 @@ struct PACKED HEFS_INDEX_NODE_DIRECTORY final { /// [0] = OFFSET /// [1] = SIZE /// @note Thus the += 2 when iterating over them. - Kernel::UInt64 fIndexNode[kHeFSBlockCount]; /// @brief Start of the index node. + Kernel::UInt64 fIndexNode[kHeFSSliceCount]; /// @brief Start of the index node. Kernel::UInt8 fColor; /// @brief Color of the node. (Red or Black). Kernel::Lba fNext, fPrev, fChild, fParent; /// @brief Red-black tree pointers. diff --git a/dev/kernel/KernelKit/PEF.h b/dev/kernel/KernelKit/PEF.h index 4c6ee5ae..03398b48 100644 --- a/dev/kernel/KernelKit/PEF.h +++ b/dev/kernel/KernelKit/PEF.h @@ -23,7 +23,7 @@ #define kPefMagicLen (5) -#define kPefVersion (3) +#define kPefVersion (4) #define kPefNameLen (256U) /* not mandatory, only for non fork based filesystems. */ @@ -59,7 +59,7 @@ enum { }; enum { - kPefSubArchAMD, + kPefSubArchAMD = 200, kPefSubArchIntel, kPefSubArchARM, kPefSubArchGeneric, @@ -67,7 +67,7 @@ enum { }; enum { - kPefKindExec = 1, /* .o */ + kPefKindExec = 1, /* .exec */ kPefKindDylib = 2, /* .dylib */ kPefKindObject = 4, /* .obj */ kPefKindDebug = 5, /* .dbg */ @@ -86,6 +86,7 @@ typedef struct PEFContainer final { UIntPtr Start; SizeT HdrSz; /* Size of header */ SizeT Count; /* container header count */ + UInt32 Checksum; } PACKED PEFContainer; /* First PEFCommandHeader starts after PEFContainer */ @@ -97,6 +98,7 @@ typedef struct PEFCommandHeader final { UInt32 Flags; /* container flags */ UInt16 Kind; /* container kind */ UIntPtr Offset; /* content offset */ + UIntPtr VMAddress; /* VM offset */ SizeT Size; /* content Size */ } PACKED PEFCommandHeader; diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 605df480..2f5bb55d 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -204,7 +204,7 @@ namespace Detail { if (dir->fKind == kHeFSFileKindDirectory) { if (KStringBuilder::Equals(dir_name, dir->fName) || KStringBuilder::Equals(dir_name, kHeFSSearchAllStr)) { - for (SizeT inode_index = 0UL; inode_index < kHeFSBlockCount; inode_index += 2) { + for (SizeT inode_index = 0UL; inode_index < kHeFSSliceCount; inode_index += 2) { mnt->fPacket.fPacketLba = dir->fIndexNode[inode_index]; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); mnt->fPacket.fPacketContent = node; @@ -329,7 +329,7 @@ namespace Detail { if (child > root->fEndIND) dirent->fChild = root->fEndIND; } - for (SizeT index = 0UL; index < (kHeFSBlockCount * 2); index += 2) { + for (SizeT index = 0UL; index < (kHeFSSliceCount * 2); index += 2) { dirent->fIndexNode[index] = root->fStartIN; dirent->fIndexNode[index + 1] = 0UL; } @@ -459,7 +459,7 @@ namespace Detail { dir->fChecksum) ke_panic(RUNTIME_CHECK_FILESYSTEM, "CRC32 failure on HeFS IND!"); - for (SizeT inode_index = 0UL; inode_index < (kHeFSBlockCount * 2); inode_index += 2) { + for (SizeT inode_index = 0UL; inode_index < (kHeFSSliceCount * 2); inode_index += 2) { if (dir->fIndexNode[inode_index] != 0) { mnt->fPacket.fPacketLba = dir->fIndexNode[inode_index]; mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE); @@ -527,7 +527,7 @@ namespace Detail { mnt->fInput(mnt->fPacket); if (KStringBuilder::Equals(dir->fName, parent_dir_name)) { - for (SizeT inode_index = 0UL; inode_index < (kHeFSBlockCount * 2); inode_index += 2) { + for (SizeT inode_index = 0UL; inode_index < (kHeFSSliceCount * 2); inode_index += 2) { if (dir->fIndexNode[inode_index] != 0) { auto lba = dir->fIndexNode[inode_index]; diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index a88b2270..25b57f56 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -83,7 +83,7 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \end{itemize} \section{Index Node Structure} -Files are stored through block links, offering native recovery fields and MIME type support. +Files are stored through slice links, offering native recovery fields and MIME type support. \begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE structure}] struct HEFS_INDEX_NODE { @@ -128,9 +128,9 @@ struct HEFS_INDEX_NODE_DIRECTORY { HeFS is designed with the following objectives: \begin{itemize} \item Red-black tree navigation for efficient directory balancing - \item Journaling fields for block-level recovery + \item Journaling fields for slice-level recovery \item Multi-encoding support: UTF-8, UTF-16, UTF-32 - \item Advanced MIME type support + \item Advanced MIME type support (Feature has been scrapped) \item Redundant fields (checksums, recovery inodes) for crash resistance \item Extensible for future LVM (Logical Volume Management) and network filesystem support \end{itemize} @@ -146,7 +146,7 @@ Planned enhancements include: \item Full journaling implementation (recovery on crash) \item Advanced ACLs (Access Control Lists) and permissions \item Logical Volume Management (LVM) integration - \item Backup Superblock and dual-boot sectors + \item Backup boot node and dual-boot sectors \item Online filesystem checking and self-healing algorithms \end{itemize} diff --git a/tooling/hefs.h b/tooling/hefs.h index 2a04a1c3..52c951ef 100644 --- a/tooling/hefs.h +++ b/tooling/hefs.h @@ -56,9 +56,6 @@ enum { kHeFSEncodingCount, }; -// Block constants -constexpr std::size_t kHeFSBlockCount = 16; - // Time type using ATime = std::uint64_t; @@ -103,54 +100,10 @@ struct alignas(8) BootNode { std::uint16_t diskStatus{}; std::uint16_t diskFlags{}; std::uint16_t vid{}; - std::uint64_t reserved{}; - std::uint64_t reserved2{}; + std::uint64_t startIN{}; + std::uint64_t endIN{}; std::uint64_t reserved3{}; std::uint64_t reserved4{}; char pad[272]{}; }; - -// Index Node -struct alignas(8) IndexNode { - char8_t name[kHeFSFileNameLen]{}; - std::uint32_t flags{}; - std::uint16_t kind{}; - std::uint32_t size{}; - std::uint32_t checksum{}; - bool symbolicLink{false}; - ATime created{}; - ATime accessed{}; - ATime modified{}; - ATime deleted{}; - std::uint32_t uid{}; - std::uint32_t gid{}; - std::uint32_t mode{}; - std::uint64_t block[kHeFSBlockCount]{}; - char pad[62]{}; -}; - -// Index Node Directory (Red-Black Tree Node) -struct alignas(8) IndexNodeDirectory { - char8_t name[kHeFSFileNameLen]{}; - std::uint32_t flags{}; - std::uint16_t kind{}; - std::uint32_t entryCount{}; - std::uint32_t checksum{}; - std::uint32_t indexNodeChecksum{}; - ATime created{}; - ATime accessed{}; - ATime modified{}; - ATime deleted{}; - std::uint32_t uid{}; - std::uint32_t gid{}; - std::uint32_t mode{}; - std::uint64_t indexNode[kHeFSBlockCount]{}; - std::uint8_t color{}; - std::uint64_t next{}; - std::uint64_t prev{}; - std::uint64_t child{}; - std::uint64_t parent{}; - char pad[32]{}; -}; - } // namespace mkfs::hefs diff --git a/tooling/mkfs.hefs.cc b/tooling/mkfs.hefs.cc index 356504f9..4d7225fe 100644 --- a/tooling/mkfs.hefs.cc +++ b/tooling/mkfs.hefs.cc @@ -96,6 +96,7 @@ int main(int argc, char** argv) { bootNode.sectorSize = kSectorSize; bootNode.startIND = start_ind; bootNode.endIND = end_ind; + bootNode.indCount = 0UL; bootNode.diskStatus = mkfs::hefs::kHeFSStatusUnlocked; std::memcpy(bootNode.magic, kHeFSMagic, kHeFSMagicLen - 1); -- cgit v1.2.3 From 1d773bed5e4193d40a47f2dcd3a88adfd1e5269b Mon Sep 17 00:00:00 2001 From: Amlal Date: Fri, 2 May 2025 10:45:40 +0200 Subject: kernel: tex: Update spec docs. Signed-off-by: Amlal --- .gitignore | 3 ++ dev/kernel/src/FS/HeFS.cc | 2 +- docs/tex/hefs.tex | 104 ++++++++++++++++++++++++---------------------- 3 files changed, 58 insertions(+), 51 deletions(-) (limited to 'docs') diff --git a/.gitignore b/.gitignore index fc656b48..4d8b3ddf 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ xcuserdata/ public/tools/*/dist/* public/frameworks/*/dist/* +*.pdf +*.aux + neoskrnl/neoskrnl.xcodeproj/project.xcworkspace/xcshareddata/ tooling/dist/mkfs.hefs diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc index 4cea6f58..d2cb403e 100644 --- a/dev/kernel/src/FS/HeFS.cc +++ b/dev/kernel/src/FS/HeFS.cc @@ -397,7 +397,7 @@ namespace Detail { hefsi_traverse_tree(tmpend, mnt, root->fStartIND, child_first, YES); } } - + dirent->fNext = tmpdir->fNext; dirent->fPrev = tmpdir->fPrev; dirent->fParent = tmpdir->fParent; diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex index 25b57f56..ba592015 100644 --- a/docs/tex/hefs.tex +++ b/docs/tex/hefs.tex @@ -33,7 +33,7 @@ \maketitle \section{Overview} -HeFS is a custom filesystem developed as part of the NeKernel project. It offers a journaling-like inode tree structure using red-black tree-inspired navigation for directories. Designed for robust use in desktop, server, and embedded environments, it supports various encoding modes and drive types. +HeFS is a high-throughput filesystem designed for NeKernel. It implements a robust directory structure based on red-black trees, uses slice-linked blocks for file storage, and includes CRC32-based integrity checks. Designed for desktop, server, and embedded contexts, it prioritizes performance, corruption recovery, and extensibility. \section{Boot Node Structure} \begin{longtable}{|l|l|p{8cm}|} @@ -41,7 +41,7 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \textbf{Field} & \textbf{Type} & \textbf{Description} \\ \hline \verb|fMagic| & \verb|char[8]| & Filesystem magic (" HeFS") \\ -\verb|fVolName| & \verb|Utf16Char[128]| & Volume name \\ +\verb|fVolName| & \verb|Utf8Char[128]| & Volume name \\ \verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0101) \\ \verb|fBadSectors| & \verb|UInt64| & Number of bad sectors detected \\ \verb|fSectorCount| & \verb|UInt64| & Total sector count \\ @@ -49,13 +49,13 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \verb|fChecksum| & \verb|UInt32| & CRC32 checksum of the boot node \\ \verb|fDiskKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\ \verb|fEncoding| & \verb|UInt8| & Encoding mode (UTF-8, UTF-16, etc.) \\ -\verb|fStartIND| & \verb|UInt64| & Starting LBA of inode tree \\ -\verb|fEndIND| & \verb|UInt64| & Ending LBA of inode tree \\ +\verb|fStartIND| & \verb|UInt64| & Starting LBA of inode directory region \\ +\verb|fEndIND| & \verb|UInt64| & Ending LBA of inode directory region \\ \verb|fINDCount| & \verb|UInt64| & Number of directory nodes allocated \\ \verb|fDiskSize| & \verb|UInt64| & Logical size of the disk \\ \verb|fDiskStatus| & \verb|UInt16| & Status of the disk (e.g., unlocked, locked) \\ \verb|fDiskFlags| & \verb|UInt16| & Disk flags (e.g., read-only) \\ -\verb|fVID| & \verb|UInt16| & Virtual ID (EPM integration) \\ +\verb|fVID| & \verb|UInt16| & Virtual ID (for EPM integration) \\ \hline \end{longtable} @@ -83,71 +83,75 @@ HeFS is a custom filesystem developed as part of the NeKernel project. It offers \end{itemize} \section{Index Node Structure} -Files are stored through slice links, offering native recovery fields and MIME type support. +The `HEFS\_INDEX\_NODE` represents a file and is constrained to 512 bytes to match hardware sector boundaries. It uses a fixed set of block pointers (slices) and CRC32 checks for data integrity. Only the local file name is stored in `fName`. -\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE structure}] +\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE (Fits 512B)}] struct HEFS_INDEX_NODE { - Utf16Char fName[256]; - UInt32 fFlags; - UInt16 fKind; - UInt32 fSize; - UInt32 fChecksum, fRecoverChecksum, fBlockChecksum, fLinkChecksum; - Utf16Char fMime[256]; - Boolean fSymLink; - ATime fCreated, fAccessed, fModified, fDeleted; - UInt32 fUID, fGID; - UInt32 fMode; - UInt64 fBlockLinkStart[16], fBlockLinkEnd[16]; - UInt64 fBlockStart[16], fBlockEnd[16]; - UInt64 fBlockRecoveryStart[16], fBlockRecoveryEnd[16]; + Utf8Char fName[256]; // Local file name + UInt32 fFlags; + UInt16 fKind; + UInt32 fSize; + UInt32 fChecksum; + + Boolean fSymLink; + ATime fCreated, fAccessed, fModified, fDeleted; + UInt32 fUID, fGID; + UInt32 fMode; + + UInt64 fBlock[16]; // Data block slices (start-only) + Char fPad[69]; // Padding to reach 512B }; \end{lstlisting} \section{Directory Node Structure} -Directories are organized into a red-black tree for efficient balancing. +Directories form a red-black tree. Each node (IND) can hold up to 16 index node references and links to its parent, siblings, and children via LBAs. -\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE\_DIRECTORY structure}] +\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE\_DIRECTORY}] struct HEFS_INDEX_NODE_DIRECTORY { - Utf16Char fName[256]; - UInt32 fFlags; - UInt16 fKind; - UInt32 fEntryCount; - UInt32 fChecksum, fIndexNodeChecksum; - Utf16Char fDim[256]; - ATime fCreated, fAccessed, fModified, fDeleted; - UInt32 fUID, fGID; - UInt32 fMode; - UInt64 fIndexNodeStart[16], fIndexNodeEnd[16]; - UInt8 fColor; - Lba fNext, fPrev, fChild, fParent; + Utf8Char fName[256]; // Directory name (not full path) + UInt32 fFlags; + UInt16 fKind; + UInt32 fEntryCount; + UInt32 fChecksum, fIndexNodeChecksum; + + ATime fCreated, fAccessed, fModified, fDeleted; + UInt32 fUID, fGID; + UInt32 fMode; + + UInt64 fIndexNode[16]; // Inode LBA references + + UInt8 fColor; // Red/Black tree color + Lba fNext, fPrev, fChild, fParent; + + Char fPad[33]; }; \end{lstlisting} -\section{Filesystem Design} +\section{Design Characteristics} -HeFS is designed with the following objectives: \begin{itemize} - \item Red-black tree navigation for efficient directory balancing - \item Journaling fields for slice-level recovery - \item Multi-encoding support: UTF-8, UTF-16, UTF-32 - \item Advanced MIME type support (Feature has been scrapped) - \item Redundant fields (checksums, recovery inodes) for crash resistance - \item Extensible for future LVM (Logical Volume Management) and network filesystem support + \item Red-black tree traversal for directory balancing + \item One-sector (512B) inode design for efficient I/O + \item Slice-linked file storage (fixed 16 slots) + \item CRC32 for boot node, inode, and directory integrity + \item Preallocated directory inodes to avoid runtime fragmentation \end{itemize} \section{Minimum Requirements} -HeFS expects a minimum disk size of 16MB. Optimal performance is aimed to be achieved with 8GB or more. -Supports: HDDs, SSDs, USB mass storage, SCSI/SAS, optical drives. +\begin{itemize} + \item Minimum disk size: 16MB + \item Recommended size: 8GB or more + \item Supported media: HDD, SSD, USB, SCSI, Flash, Optical +\end{itemize} \section{Future Work} -Planned enhancements include: \begin{itemize} - \item Full journaling implementation (recovery on crash) - \item Advanced ACLs (Access Control Lists) and permissions - \item Logical Volume Management (LVM) integration - \item Backup boot node and dual-boot sectors - \item Online filesystem checking and self-healing algorithms + \item Journaling layer for recovery + \item Extended access control and ACL support + \item Logical Volume Management (LVM) + \item Dual boot node layout for redundancy + \item Self-healing and online fsck tools \end{itemize} \end{document} -- cgit v1.2.3