summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorAmlal El Mahrouss <amlal@nekernel.org>2025-04-21 08:47:04 +0200
committerAmlal El Mahrouss <amlal@nekernel.org>2025-04-21 08:47:04 +0200
commit00e314410e6049c2ddbcb4861c04af6b06eeeea3 (patch)
treebe4ee060de121ac0b825b5b0263deb3a0ac74619
parentf00211b6023ad406553a6bf9208092f834a44cdd (diff)
dev, kernel, tools, tooling, tex: add mk_app tool, see details.
- Patch HeFS implementation file, working on a allocation function now. - Generated LaTeX specs from source code. - Add mk.{hefs, nefs} tools for future formatting (with diutil first) - Add python tool to generate user apps for NeKernel. Signed-off-by: Amlal El Mahrouss <amlal@nekernel.org>
-rw-r--r--dev/kernel/src/FS/HeFS.cc82
-rw-r--r--docs/tex/hefs.tex121
-rw-r--r--public/tools/diutil/src/CommandLine.cc6
-rw-r--r--public/tools/mk.hefs/.keep0
-rw-r--r--public/tools/mk.hefs/mk.hefs.json12
-rw-r--r--public/tools/mk.hefs/src/CommandLine.cc14
-rw-r--r--public/tools/mk.nefs/.keep0
-rw-r--r--public/tools/mk.nefs/mk.nefs.json12
-rw-r--r--public/tools/mk.nefs/src/CommandLine.cc14
-rwxr-xr-xtooling/mk_app.py67
-rwxr-xr-xtooling/mk_img.py (renamed from tooling/copy_to_fat32.py)3
11 files changed, 284 insertions, 47 deletions
diff --git a/dev/kernel/src/FS/HeFS.cc b/dev/kernel/src/FS/HeFS.cc
index 142ddbea..8d73b61e 100644
--- a/dev/kernel/src/FS/HeFS.cc
+++ b/dev/kernel/src/FS/HeFS.cc
@@ -15,6 +15,7 @@
#include <NewKit/KString.h>
#include <NewKit/Utils.h>
#include <FirmwareKit/EPM.h>
+#include <FirmwareKit/GPT.h>
#include <KernelKit/ProcessScheduler.h>
#include <KernelKit/User.h>
@@ -22,7 +23,30 @@ namespace Kernel
{
namespace Detail
{
- STATIC HEFS_INDEX_NODE* hefs_get_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept
+ /// @brief Forward declarations of internal functions.
+
+ /// @brief Get the index node of a file or directory.
+ /// @param root The root node of the filesystem.
+ /// @param mnt The drive to read from.
+ /// @param dir_name The name of the directory.
+ /// @param file_name The name of the file.
+ /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown).
+ STATIC HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept;
+
+ /// @brief Allocate a new index node.
+ /// @param root The root node of the filesystem.
+ /// @param mnt The drive to read from.
+ /// @param parent_dir_name The name of the parent directory.
+ /// @return Status, see err_global_get().
+ STATIC BOOL hefs_allocate_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* parent_dir_name, HEFS_INDEX_NODE* node) noexcept;
+
+ /// @brief Get the index node of a file or directory.
+ /// @param root The root node of the filesystem.
+ /// @param mnt The drive to read from.
+ /// @param dir_name The name of the directory.
+ /// @param file_name The name of the file.
+ /// @param kind The kind of the file (regular, directory, block, character, FIFO, socket, symbolic link, unknown).
+ STATIC HEFS_INDEX_NODE* hefs_fetch_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* dir_name, const Utf16Char* file_name, UInt8 kind) noexcept
{
if (root)
{
@@ -42,10 +66,9 @@ namespace Kernel
while (start != end)
{
- if (hop_watch++ > 100)
+ if (hop_watch > 100)
{
- kout << "Error: Hop watch exceeded.\r";
-
+ kout << "Error: Hop watch exceeded, filesystem is stalling.\r";
break;
}
@@ -66,8 +89,10 @@ namespace Kernel
delete node;
delete dir;
- err_global_get() = kErrorFileNotFound;
+ dir = nullptr;
+ node = nullptr;
+ err_global_get() = kErrorFileNotFound;
return nullptr;
}
@@ -78,9 +103,10 @@ namespace Kernel
{
for (SizeT inode_index = 0UL; inode_index < kHeFSBlockCount; ++inode_index)
{
- if (dir->fIndexNodeStart[inode_index] != 0)
+ if (dir->fIndexNodeStart[inode_index] != 0 ||
+ dir->fIndexNodeEnd[inode_index] != 0)
{
- mnt->fPacket.fPacketLba = dir->fIndexNodeStart[inode_index];
+ mnt->fPacket.fPacketLba = (!dir->fIndexNodeStart[inode_index]) ? dir->fIndexNodeEnd[inode_index] : dir->fIndexNodeStart[inode_index];
mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE);
mnt->fPacket.fPacketContent = node;
@@ -91,35 +117,22 @@ namespace Kernel
if (KStringBuilder::Equals(file_name, node->fName) && node->fKind == kind)
{
delete dir;
+ dir = nullptr;
return node;
}
}
else
{
- break;
- }
- }
- else if (dir->fIndexNodeEnd[inode_index] != 0)
- {
- mnt->fPacket.fPacketLba = dir->fIndexNodeEnd[inode_index];
- mnt->fPacket.fPacketSize = sizeof(HEFS_INDEX_NODE);
- mnt->fPacket.fPacketContent = node;
+ err_global_get() = kErrorDiskIsCorrupted;
- mnt->fInput(mnt->fPacket);
+ delete dir;
+ delete node;
- if (mnt->fPacket.fPacketGood)
- {
- if (KStringBuilder::Equals(file_name, node->fName) && node->fKind == kind)
- {
- delete dir;
+ dir = nullptr;
+ node = nullptr;
- return node;
- }
- }
- else
- {
- break;
+ return nullptr;
}
}
}
@@ -148,6 +161,21 @@ namespace Kernel
return nullptr;
}
+
+ /// @brief Allocate a new index node.
+ /// @param root The root node of the filesystem.
+ /// @param mnt The drive to read from.
+ /// @param parent_dir_name The name of the parent directory.
+ /// @return Status, see err_global_get().
+ STATIC BOOL hefs_allocate_index_node(HEFS_BOOT_NODE* root, DriveTrait* mnt, const Utf16Char* parent_dir_name, HEFS_INDEX_NODE* node) noexcept
+ {
+ NE_UNUSED(root);
+ NE_UNUSED(mnt);
+ NE_UNUSED(parent_dir_name);
+ NE_UNUSED(node);
+
+ return NO;
+ }
} // namespace Detail
} // namespace Kernel
diff --git a/docs/tex/hefs.tex b/docs/tex/hefs.tex
index e74cf5e3..3370af48 100644
--- a/docs/tex/hefs.tex
+++ b/docs/tex/hefs.tex
@@ -1,32 +1,123 @@
\documentclass{article}
-\usepackage[a4paper,margin=1in]{geometry}
+\usepackage[a4paper, margin=1in]{geometry}
+\usepackage{amsmath, amssymb}
\usepackage{listings}
\usepackage{xcolor}
-\usepackage{amsmath}
-\usepackage{hyperref}
-\usepackage{longtable}
-\usepackage{titlesec}
-\usepackage{fancyhdr}
-\usepackage{caption}
\usepackage{graphicx}
+\usepackage{enumitem}
+\usepackage{caption}
+\usepackage{longtable}
-\definecolor{codegray}{gray}{0.95}
-\lstset{
- backgroundcolor=\color{codegray},
+\definecolor{lightgray}{gray}{0.95}
+
+\lstdefinestyle{cstyle}{
+ language=C++,
+ backgroundcolor=\color{lightgray},
basicstyle=\ttfamily\small,
+ keywordstyle=\color{blue},
+ commentstyle=\color{green!60!black},
+ stringstyle=\color{orange},
+ numbers=left,
+ numberstyle=\tiny,
breaklines=true,
frame=single,
- tabsize=4,
- language=C++,
showstringspaces=false
}
-\title{HeFS: High-Throughput Extended File System Specification}
+\title{HeFS (Hierarchical Embedded File System) Specification}
\author{Amlal El Mahrouss}
-\date{2025}
+\date{2024–2025}
\begin{document}
\maketitle
-\end{document}
+\section{Overview}
+HeFS is a custom filesystem developed as part of the NeKernel project. It offers a journaling-like inode tree structure using red-black tree-inspired navigation for directories. Designed for robust use in desktop and server workloads, it supports various encoding modes and drive types.
+
+\section{Boot Node Structure}
+\begin{longtable}{|l|l|p{8cm}|}
+\hline
+\textbf{Field} & \textbf{Type} & \textbf{Description} \\
+\hline
+\verb|fMagic| & \verb|char[8]| & Filesystem magic (" HeFS") \\
+\verb|fVolName| & \verb|Utf16Char[128]| & Volume name \\
+\verb|fVersion| & \verb|UInt32| & Filesystem version (e.g., 0x0100) \\
+\verb|fSectorCount| & \verb|UInt64| & Total sector count \\
+\verb|fSectorSize| & \verb|UInt64| & Size of each sector \\
+\verb|fDriveKind| & \verb|UInt8| & Type of drive (e.g., HDD, SSD, USB) \\
+\verb|fEncoding| & \verb|UInt8| & Encoding mode (UTF-8, UTF-16, etc.) \\
+\verb|fStartIND| & \verb|UInt64| & Starting LBA of inode tree \\
+\verb|fEndIND| & \verb|UInt64| & Ending LBA of inode tree \\
+\verb|fChecksum| & \verb|UInt32| & CRC32 of boot node \\
+\verb|fDiskSize| & \verb|UInt64| & Logical size of the disk \\
+\hline
+\end{longtable}
+
+\section{File Types and Flags}
+\subsection*{File Kinds}
+\begin{itemize}[label=--]
+ \item \verb|0x00| — Regular File
+ \item \verb|0x01| — Directory
+ \item \verb|0x02| — Block Device
+ \item \verb|0x03| — Character Device
+ \item \verb|0x04| — FIFO
+ \item \verb|0x05| — Socket
+ \item \verb|0x06| — Symbolic Link
+\end{itemize}
+
+\subsection*{Drive Types}
+\begin{itemize}[label=--]
+ \item \verb|0xC0| — Hard Drive
+ \item \verb|0xC1| — SSD
+ \item \verb|0xCC| — Mass Storage Device (USB)
+\end{itemize}
+
+\section{Index Node Structure}
+The inode tree allows for fast access and recovery via dual start/end block mappings and timestamped metadata.
+
+\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE structure}]
+struct HEFS_INDEX_NODE {
+ Utf16Char fName[256];
+ UInt32 fFlags;
+ UInt16 fKind;
+ UInt32 fSize;
+ UInt32 fChecksum, fRecoverChecksum, fBlockChecksum, fLinkChecksum;
+ ATime fCreated, fAccessed, fModified, fDeleted;
+ UInt32 fUID, fGID;
+ UInt32 fMode;
+ UInt64 fBlockLinkStart[16], fBlockLinkEnd[16];
+ UInt64 fBlockStart[16], fBlockEnd[16];
+ UInt64 fBlockRecoveryStart[16], fBlockRecoveryEnd[16];
+};
+\end{lstlisting}
+
+\section{Directory Node Structure}
+Each directory is a red-black tree node with child and sibling pointers.
+
+\begin{lstlisting}[style=cstyle, caption={HEFS\_INDEX\_NODE\_DIRECTORY structure}]
+struct HEFS_INDEX_NODE_DIRECTORY {
+ Utf16Char fName[256];
+ UInt32 fFlags;
+ UInt16 fKind;
+ UInt32 fSize;
+ UInt32 fChecksum, fIndexNodeChecksum;
+ ATime fCreated, fAccessed, fModified, fDeleted;
+ UInt32 fUID, fGID;
+ UInt32 fMode;
+ UInt64 fIndexNodeStart[16], fIndexNodeEnd[16];
+ UInt8 fColor;
+ Lba fNext, fPrev, fChild, fParent;
+};
+\end{lstlisting}
+
+\section{Future Work}
+Planned extensions include:
+\begin{itemize}
+ \item Journaling recovery logic
+ \item Advanced permission flags
+ \item Logical volume management support
+ \item Cross-filesystem linking (EPM integration)
+\end{itemize}
+
+\end{document} \ No newline at end of file
diff --git a/public/tools/diutil/src/CommandLine.cc b/public/tools/diutil/src/CommandLine.cc
index 604520a9..3e7353e6 100644
--- a/public/tools/diutil/src/CommandLine.cc
+++ b/public/tools/diutil/src/CommandLine.cc
@@ -9,12 +9,12 @@
#include <DiskImage.fwrk/headers/DiskImage.h>
-static const Char kDiskName[kDIDiskNameLen] = "Disk";
-static SInt32 kDiskSectorSz = 512;
+static const Char kDiskName[kDIDiskNameLen] = "DISK";
+static SInt32 kDiskSectorSz = kDISectorSz;
static SizeT kDiskSz = gib_cast(4);
static const Char kOutDisk[kDIOutNameLen] = "disk.eimg";
-/// @brief Filesystem tool entrypoint.
+/// @brief Disk Utility entrypoint.
int main(int argc, char** argv)
{
for (SInt32 arg = 0; arg < argc; ++arg)
diff --git a/public/tools/mk.hefs/.keep b/public/tools/mk.hefs/.keep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/public/tools/mk.hefs/.keep
diff --git a/public/tools/mk.hefs/mk.hefs.json b/public/tools/mk.hefs/mk.hefs.json
new file mode 100644
index 00000000..03f925cf
--- /dev/null
+++ b/public/tools/mk.hefs/mk.hefs.json
@@ -0,0 +1,12 @@
+{
+ "compiler_path": "g++",
+ "compiler_std": "c++20",
+ "headers_path": ["./", "../../../dev"],
+ "sources_path": ["src/CommandLine.cc"],
+ "output_name": "./dist/mk.hefs",
+ "cpp_macros": [
+ "kCCVersion=0x0100",
+ "kCCVersionHighest=0x0100",
+ "kCCVersionLowest=0x0100"
+ ]
+}
diff --git a/public/tools/mk.hefs/src/CommandLine.cc b/public/tools/mk.hefs/src/CommandLine.cc
new file mode 100644
index 00000000..23681be8
--- /dev/null
+++ b/public/tools/mk.hefs/src/CommandLine.cc
@@ -0,0 +1,14 @@
+/*
+ * Created on Thu Oct 17 08:00:42 CEST 2024
+ *
+ * Copyright (c) 2025 Amlal El Mahrouss
+ */
+
+#include <user/SystemCalls.h>
+
+/// @brief Placeholder program.
+
+SInt32 main(SInt32 argc, Char* argv[])
+{
+ return EXIT_FAILURE;
+}
diff --git a/public/tools/mk.nefs/.keep b/public/tools/mk.nefs/.keep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/public/tools/mk.nefs/.keep
diff --git a/public/tools/mk.nefs/mk.nefs.json b/public/tools/mk.nefs/mk.nefs.json
new file mode 100644
index 00000000..e92e0b9c
--- /dev/null
+++ b/public/tools/mk.nefs/mk.nefs.json
@@ -0,0 +1,12 @@
+{
+ "compiler_path": "g++",
+ "compiler_std": "c++20",
+ "headers_path": ["./", "../../../dev"],
+ "sources_path": ["src/CommandLine.cc"],
+ "output_name": "./dist/mk.nefs",
+ "cpp_macros": [
+ "kCCVersion=0x0100",
+ "kCCVersionHighest=0x0100",
+ "kCCVersionLowest=0x0100"
+ ]
+}
diff --git a/public/tools/mk.nefs/src/CommandLine.cc b/public/tools/mk.nefs/src/CommandLine.cc
new file mode 100644
index 00000000..23681be8
--- /dev/null
+++ b/public/tools/mk.nefs/src/CommandLine.cc
@@ -0,0 +1,14 @@
+/*
+ * Created on Thu Oct 17 08:00:42 CEST 2024
+ *
+ * Copyright (c) 2025 Amlal El Mahrouss
+ */
+
+#include <user/SystemCalls.h>
+
+/// @brief Placeholder program.
+
+SInt32 main(SInt32 argc, Char* argv[])
+{
+ return EXIT_FAILURE;
+}
diff --git a/tooling/mk_app.py b/tooling/mk_app.py
new file mode 100755
index 00000000..34a7702e
--- /dev/null
+++ b/tooling/mk_app.py
@@ -0,0 +1,67 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import json
+import sys
+
+def create_directory_structure(base_path, project_name):
+ # Define the directory structure
+ structure = {
+ project_name: {
+ "src": {
+ ".keep": None
+ },
+ "vendor": {
+ ".keep": None
+ },
+ ".keep": None,
+ f"{project_name}.json": {}
+ }
+ }
+
+ def create_structure(path, structure):
+ for name, content in structure.items():
+ current_path = os.path.join(path, name)
+ # Create directories or files based on the content type
+ if isinstance(content, dict) and current_path.endswith(".json") == False:
+ os.makedirs(current_path, exist_ok=True)
+ create_structure(current_path, content)
+ elif content is None:
+ # Create an empty file
+ with open(current_path, 'w') as f:
+ pass
+
+ # Create the base directory
+ os.makedirs(base_path, exist_ok=True)
+ create_structure(base_path, structure)
+
+ # Create the JSON file
+ diutil_json_path = os.path.join(base_path, project_name, f"{project_name}.json")
+ manifest = {
+ "compiler_path": "g++",
+ "compiler_std": "c++20",
+ "headers_path": ["./", "../../../dev/kernel", "../../../public/frameworks/", "../../../dev/", "./"],
+ "sources_path": [
+
+ ],
+ "output_name": f"./dist/{project_name}",
+ "cpp_macros": [
+ "kSampleVersion=0x0100",
+ "kSampleVersionHighest=0x0100",
+ "kSampleVersionLowest=0x0100",
+ "__NE_SDK__"
+ ]
+ }
+
+ with open(diutil_json_path, 'w') as json_file:
+ json.dump(manifest, json_file, indent=4)
+
+if __name__ == "__main__":
+ if len(sys.argv) != 2:
+ print("Usage: mk_app.py <project_name>")
+ sys.exit(1)
+
+ base_path = os.getcwd() # Use the current working directory as the base path
+ create_directory_structure(base_path, sys.argv[1])
+ print("Ne application created successfully.") \ No newline at end of file
diff --git a/tooling/copy_to_fat32.py b/tooling/mk_img.py
index ea0cb1b9..253e5a6c 100755
--- a/tooling/copy_to_fat32.py
+++ b/tooling/mk_img.py
@@ -17,7 +17,6 @@ def copy_to_fat(image_path, source_dir):
try:
files_to_copy = glob.glob(os.path.join(source_dir, "*"))
-
# Now build the command
command = ["mcopy", "-spom", "-i", image_path] + files_to_copy + ["::"]
subprocess.run(command, check=True)
@@ -35,7 +34,7 @@ def copy_to_fat(image_path, source_dir):
if __name__ == "__main__":
if len(sys.argv) != 3:
- print("Usage: python3 copy_to_fat32.py <fat32_image> <source_directory>")
+ print("Usage: mk_img.py <fat32_image> <source_directory>")
sys.exit(1)
image_path = sys.argv[1]