From d10b181547fdf61b0086a2d062ac01eaf5961401 Mon Sep 17 00:00:00 2001 From: Jan Philipp Hafer Date: Wed, 15 Jan 2025 19:56:41 +0100 Subject: [PATCH] more stuff, including sanitizer scalability --- .config/nvim/lua/my_fmt.lua | 5 +- .config/nvim/lua/my_lint.lua | 4 +- .config/nvim/lua/my_lsp.lua | 6 +- build.zig | 5 +- src/CMakeLists.txt | 1 + src/smartcv.cls | 37 ++++++----- src/smartcv_example.tex | 102 +++++++++++++++-------------- templates/ai_tools | 3 + templates/common.c | 11 ++-- templates/common.c_and_cpp | 55 +++++++++++----- templates/common.cpp | 8 ++- templates/common.css | 24 +++++++ templates/common.tex | 5 ++ templates/cpu | 7 +- templates/formal_modeling.txt | 7 ++ templates/gpu | 9 +++ templates/kernel_semantics | 7 +- templates/parallel_programming.txt | 18 ++--- templates/security | 2 + templates/testing | 7 +- templates/tooling | 5 ++ 21 files changed, 215 insertions(+), 113 deletions(-) diff --git a/.config/nvim/lua/my_fmt.lua b/.config/nvim/lua/my_fmt.lua index 5a2f532..06dade5 100644 --- a/.config/nvim/lua/my_fmt.lua +++ b/.config/nvim/lua/my_fmt.lua @@ -17,6 +17,7 @@ if not has_conform then return end -- ruff_format: no cmd for combined linting and fmt yet https://github.com/astral-sh/ruff/issues/8232 -- stylua: cargo install stylua --features lua52 +-- tex-fmt: cargo install tex-fmt local fmts_by_ft = { -- lsp_format = 'never(default)|fallback|prefer|first|last' -- cmake = { 'cmake_format' }, @@ -25,8 +26,10 @@ local fmts_by_ft = { lua = { 'stylua' }, -- stylua: ignore start|end -- python = { 'ruff_format' }, -- # fmt: off|on, # fmt: skip rust = { 'rustfmt', lsp_format = 'prefer' }, - sh = { 'shfmt' }, + -- sh = { 'shfmt' }, -- go not great language shtml = { 'superhtml' }, + -- https://github.com/WGUNDERWOOD/tex-fmt/issues/55 + -- tex = { 'tex-fmt' }, -- % tex-fmt: off|on, % tex-fmt: skip zig = { 'zigfmt', lsp_format = 'prefer' }, -- -- zig = { lsp_format = 'prefer' }, ziggy = { 'ziggy' }, diff --git a/.config/nvim/lua/my_lint.lua b/.config/nvim/lua/my_lint.lua index 591a3fb..3f4ad46 100644 --- a/.config/nvim/lua/my_lint.lua +++ b/.config/nvim/lua/my_lint.lua @@ -24,6 +24,8 @@ if not ok_lint then return end -- uv tool install weasyprint -- uv tool install yt-dlp +-- idea https://github.com/codex-semantics-library/codex + --==installing pixi -- git clone 'https://github.com/prefix-dev/pixi' -- cargo install --locked --git https://github.com/prefix-dev/pixi.git pixi @@ -54,7 +56,7 @@ lint.linters_by_ft = { -- // NOLINTEND -- // NOLINTBEGIN(errorclass) -- somecode // NOLINT - c = { 'clangtidy' }, + c = { 'clangtidy' }, -- codex cpp = { 'clangtidy' }, -- luacheck: push ignore -- luacheck: pop ignore diff --git a/.config/nvim/lua/my_lsp.lua b/.config/nvim/lua/my_lsp.lua index 0b38b0a..86faf54 100644 --- a/.config/nvim/lua/my_lsp.lua +++ b/.config/nvim/lua/my_lsp.lua @@ -33,10 +33,10 @@ local aucmd_lsp = vim.api.nvim_create_augroup('aucmds_lsp', { clear = true }) -- 'bashls', -- 'bash-language-server' -- 'biome' -- 'biome' -- 'jedi_language_server', -- 'jedi-language-server' prefer ruffs server --- 'ltex', -- 'ltex-ls' -- 'clangd', -- 'clangd' -- 'lemminx', -- 'lemminx' -- 'omnisharp' -- 'omnisharp' +-- 'texlab' -- 'texlab' -- -- pip3 install -U --user jedi-language-server -- pipx install jedi-language-server @@ -135,12 +135,12 @@ lspconfig.texlab.setup { on_attach = common_on_attach, settings = { texlab = { - auxDirectory = { 'build' }, build = { --args = { '-pdflatex=lualatex', '-pdf', '-interaction=nonstopmode', '-synctex=1', '%f' } --args = { '-pdflatex=lualatex', '-pdf', '-interaction=nonstopmode', '-synctex=1', '-outdir=build', 'main.tex' } - args = { '-pdf', '-interaction=nonstopmode', '-synctex=1', '-outdir=build', 'main.tex' }, + args = { '-pdf', '-interaction=nonstopmode', '-synctex=1', '-outdir=build', '%f' }, }, + latexFormatter = 'none', }, }, } diff --git a/build.zig b/build.zig index 7d0d009..a7330c2 100644 --- a/build.zig +++ b/build.zig @@ -22,7 +22,7 @@ pub fn build(b: *std.Build) !void { // Ps1 | - | - | - | - | - // Py | - | - | - | - | - // Rs | - | - | - | - | - - // Sh | y | y | n | n | n + // Sh | n | y | n | n | n // Tex | n | n | n | y | n // Zig | y | y | n | y | y @@ -312,7 +312,8 @@ fn lintLua(b: *std.Build, run_step: *std.Build.Step) void { // fn checkRs() void {} // nofmt nolint nobuild noproj fn fmtSh(b: *std.Build, run_step: *std.Build.Step) void { - // shfmt has no way to disable fmt / check mode, so it is not enabled + // go bad + // shfmt has no way to disable fmt / has no check mode, so it is not enabled _ = b; _ = run_step; // for (SingleShFiles[0..]) |shfile| { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d18b99f..30229a0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ # toolchain. Reason: # # Only `libc++` is supported +# See also https://gitlab.kitware.com/cmake/cmake/-/issues/25965 # set CXXFLAGS -stdlib=libc++ # CXXFLAGS=-stdlib=libc++ diff --git a/src/smartcv.cls b/src/smartcv.cls index 716e4eb..b868a4b 100644 --- a/src/smartcv.cls +++ b/src/smartcv.cls @@ -9,10 +9,10 @@ \usepackage{hyperref} \hypersetup{ - colorlinks=true, - linkcolor=blue, - filecolor=magenta, - urlcolor=blue, + colorlinks=true, + linkcolor=blue, + filecolor=magenta, + urlcolor=blue, } \pagestyle{empty} % no page numbers @@ -22,7 +22,8 @@ \def \nameskip {\smallskip} % space after name \def \addresssep {$\diamond$} % const \def \addressskip {\smallskip} % space between two address lines -\def \rSectionlineskip {\smallskip} % space above horizontal line for each section +\def \rSectionlineskip {\smallskip} % space above horizontal line for +% each section \def \rSectionskip {\smallskip} % space after heading section \let \@name \relax % var @@ -36,8 +37,8 @@ % \printaddress to style address line \def \printaddress #1{ \begingroup - \def \\ {\addresssep\ } - \centerline{#1} + \def \\ {\addresssep\ } + \centerline{#1} \endgroup \par \addressskip @@ -46,8 +47,8 @@ % \printname to print name as page header \def \printname { \begingroup - \hfil{\MakeUppercase{\namesize\bf \@name}}\hfil - \nameskip\break + \hfil{\MakeUppercase{\namesize\bf \@name}}\hfil + \nameskip\break \endgroup } @@ -56,18 +57,18 @@ \renewcommand{\document}{ \ori@document % document start \@ifundefined{@name} { \@latex@error - {Variable '@name' is undefined.} - {Use \ name{Name Surname}.} + {Variable '@name' is undefined.} + {Use \ name{Name Surname}.} }{ \printname } \@ifundefined{@addressOne} { \@latex@error - {Variable '@addressOne' is undefined.} - {Use \ addrone{+phone \\ place}.} + {Variable '@addressOne' is undefined.} + {Use \ addrone{+phone \\ place}.} }{ \printaddress{\@addressOne} } \@ifundefined{@addressTwo} { \@latex@error - {Variable '@addressTwo' is undefined.} - {Use \ addrtwo{\href{link1}{descr1} \\ ..}.} + {Variable '@addressTwo' is undefined.} + {Use \ addrtwo{\href{link1}{descr1} \\ ..}.} }{ \printaddress{\@addressTwo} } } @@ -79,9 +80,9 @@ \rSectionlineskip \hrule % horizontal line \begin{list}{}{ % list for each section item - \setlength{\leftmargin}{0cm} % margin within section - } + \setlength{\leftmargin}{0cm} % margin within section + } \item[] -}{ + }{ \end{list} } diff --git a/src/smartcv_example.tex b/src/smartcv_example.tex index 0f4c2d3..a765829 100644 --- a/src/smartcv_example.tex +++ b/src/smartcv_example.tex @@ -1,5 +1,9 @@ % SPDX-License-Identifier: 0BSD -% latexmk -pvc -pdflatex='lualatex --file-line-error --synctex=1' -pdf -outdir=build src/smartcv_example.tex +% dont use, breaks on %-strings in href, see https://github.com/WGUNDERWOOD/tex-fmt/issues/55 +% tex-fmt src/smartcv_example.tex +% cargo install tex-fmt; tex-fmt src/smartcv_example.tex +% latexmk -pvc -pdflatex='lualatex --file-line-error --synctex=1' +% -pdf -outdir=build src/smartcv_example.tex % LuaTEX Reference Manual % https://en.wikibooks.org/wiki/TeX % idea: docs on internal commands like \@ifundefined \@latex@error @@ -9,7 +13,8 @@ \usepackage[left=1cm,top=1cm,right=1cm,bottom=1cm]{geometry} % Document margins \usepackage{enumitem} % \begin{itemize}[noitemsep, notopsep=0pt] \end{itemize} % itemize adjustments <<<< Adjust here -\setlist{nosep,topsep=-\parskip} % eliminate itemize space without changing \parskip, dep enumitem +\setlist{nosep,topsep=-\parskip} % eliminate itemize space without +% changing \parskip, dep enumitem \name{Name Surname} % Your name \addrone{+phone \\ place} @@ -19,81 +24,82 @@ \href{https://example.invalid}{example.invalid} \\ \href{https://github.com/invalid}{github.com/invalid} } % -%Testing, Formal Methods, Site Reliability Engineering, Observability, and Infrastructure. +%Testing, Formal Methods, Site Reliability Engineering, +% Observability, and Infrastructure. \begin{document} %OBJECTIVE \begin{rSection}{OBJECTIVE} -Software Engineer with X+ years of work experience in something1, something2 -and something3 seeking full-time something roles focused on cool stuff to do. + Software Engineer with X+ years of work experience in something1, something2 + and something3 seeking full-time something roles focused on cool stuff to do. \end{rSection} %TECHNICAL_STRENGTHS \begin{rSection}{SKILLS} -\begin{tabular}{ @{} >{\bfseries}l @{\hspace{6ex}} l } - Technical Skills & BuildOps, DevOps, Some1, Some2, Some3 -\\ -Soft Skills & How character fits to the job, customer or team \\ -Low level languages & C, Some1, Some2, Some3\\ -High level languages & Some1, Some2, Various Shells\\ -\end{tabular}\\ + \begin{tabular}{ @{} >{\bfseries}l @{\hspace{6ex}} l } + Technical Skills & BuildOps, DevOps, Some1, Some2, Some3 + \\ + Soft Skills & How character fits to the job, customer or team \\ + Low level languages & C, Some1, Some2, Some3\\ + High level languages & Some1, Some2, Various Shells\\ + \end{tabular}\\ \end{rSection} \begin{rSection}{WORK EXPERIENCE} -\textbf{Software Engineer Y} \hfill Nov 20XY - present\\ -company1 \hfill \textit{place1} -\begin{itemize} - % \itemsep -2pt {} - \item brag item1 - \item brag item2 - \item brag item3 -\end{itemize} - -\textbf{Software Engineer X} \hfill Jun 20XY - Oct 20XY\\ -company2 \hfill \textit{place2} -\begin{itemize} - % \itemsep -2pt {} - \item brag item1 - \item brag item2 - \item brag item3 -\end{itemize} + \textbf{Software Engineer Y} \hfill Nov 20XY - present\\ + company1 \hfill \textit{place1} + \begin{itemize} + % \itemsep -2pt {} + \item brag item1 + \item brag item2 + \item brag item3 + \end{itemize} + + \textbf{Software Engineer X} \hfill Jun 20XY - Oct 20XY\\ + company2 \hfill \textit{place2} + \begin{itemize} + % \itemsep -2pt {} + \item brag item1 + \item brag item2 + \item brag item3 + \end{itemize} \end{rSection} %====EDUCATION \begin{rSection}{Education} -{\bf Master of Computer Science}, university \hfill {Oct 20XY - Apr 20XY}\\ -Master Thesis on description (tool), Focus Embedded and Medicine\\ -{\bf Bachelor of Computer Science}, university \hfill {Oct 20XY - Mar 20XY}\\ -Bachelor Thesis on description (tool) -%Minor in Linguistics \smallskip \\ -%Member of Eta Kappa Nu \\ -%Member of Upsilon Pi Epsilon \\ + {\bf Master of Computer Science}, university \hfill {Oct 20XY - Apr 20XY}\\ + Master Thesis on description (tool), Focus Embedded and Medicine\\ + {\bf Bachelor of Computer Science}, university \hfill {Oct 20XY - Mar 20XY}\\ + Bachelor Thesis on description (tool) + %Minor in Linguistics \smallskip \\ + %Member of Eta Kappa Nu \\ + %Member of Upsilon Pi Epsilon \\ \end{rSection} %====OPEN_SOURCE_WORK_AND_PROJECTS \begin{rSection}{OPEN SOURCE WORK AND PROJECTS} -% \vspace{-1.25em} -\begin{itemize} -% \itemsep -2pt {} - \item brag item1 - \item brag item2 - \item brag item3 -\end{itemize} + % \vspace{-1.25em} + \begin{itemize} + % \itemsep -2pt {} + \item brag item1 + \item brag item2 + \item brag item3 + \end{itemize} \end{rSection} %====TEACHING_AND_CONSULTING \begin{rSection}{TEACHING AND CONSULTING} -\begin{itemize} - \item brag item1 - \item brag item2 - \item brag item3 -\end{itemize} + \begin{itemize} + \item brag item1 + \item brag item2 + \item brag item3 + \end{itemize} \end{rSection} diff --git a/templates/ai_tools b/templates/ai_tools index 79008cd..dc68fdb 100644 --- a/templates/ai_tools +++ b/templates/ai_tools @@ -9,3 +9,6 @@ TODO Train writing short mails. fish prompt, no idea how useful https://github.com/Realiserad/fish-ai + +https://haystack.deepset.ai/ +pdftotext: simplify table of content, use known keywords, use llm for synonyms diff --git a/templates/common.c b/templates/common.c index 3b391fe..7ce80e2 100644 --- a/templates/common.c +++ b/templates/common.c @@ -393,11 +393,12 @@ size_t getFileSize(char const *file_path) { file = fopen(file_path, "rb"); if (file == NULLPTR) exit(1); // could not open file, better use file handle - fseek(file, 0u, SEEK_END); + (void)fseek(file, 0u, SEEK_END); long file_size_or_err = ftell(file); if (file_size_or_err < 0) exit(1); // invalid file size - fseek(file, 0u, SEEK_SET); + (void)fseek(file, 0u, SEEK_SET); + (void)fclose(file); return (size_t)file_size_or_err; } @@ -1018,8 +1019,10 @@ void ape_win_print(void) { #ifndef _WIN32 void ape_print(void) { FILE *f1 = fopen("file1", "a+"); - fprintf(f1, "sometext\n"); - fclose(f1); + if (f1 != NULLPTR) { + fprintf(f1, "sometext\n"); + fclose(f1); + } } #endif diff --git a/templates/common.c_and_cpp b/templates/common.c_and_cpp index 6742407..05c35a9 100644 --- a/templates/common.c_and_cpp +++ b/templates/common.c_and_cpp @@ -26,42 +26,61 @@ cc -O2 -dM -E - < /dev/null 4. discouraged search paths -5. sanitizers -instrumented sanitizers with 2x-4x slodown vs dynamic ones with 20x-50x. --fsanitize=address --fsanitize=leak --fsanitize=thread --fsanitize=undefined --fsanitize=type --O1 -g -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-common +5. Sanitizers +Instrumented sanitizers with 2x-4x slowdown vs dynamic ones with 20x-50x. +Nr | Clang usage | Zig usage | Memory | Runtime | Comments +-- | ---------------------------- | ----------------- | ---------------- | -------- | -------- +1 | -fsanitize=address | alloc + safety | 1x (3x stack) | 2x | Clang 16+ TB of virt mem +2 | -fsanitize=leak | allocator | 1x | 1x | on exit ?x? more mem+time +3 | -fsanitize=memory | unimplemented | 2-3x | 3x | +4 | -fsanitize=thread | -fsanitize=thread | 5-10x+1MB/thread | 5-15x | Clang ?x? ("lots of") virt mem +5 | -fsanitize=type | unimplemented | ? | ? | not enough data +6 | -fsanitize=undefined | safety mode | 1x | ~1x | +7 | -fsanitize=dataflow | unimplemented | 1-2x? | 1-4x? | wip, get variable dependencies +8 | -fsanitize=memtag | unimplemented | ~1.0Yx? | ~1.0Yx? | wip, address cheri-like ptr tagging +9 | -fsanitize=cfi | unimplemented | 1x | ~1x | forward edge ctrl flow protection +10 | -fsanitize=safe-stack | unimplemented | 1x | ~1x | backward edge ctrl flow protection +11 | -fsanitize=shadow-call-stack | unimplemented | 1x | ~1x | backward edge ctrl flow protection + +Sanitizers 1-6 are recommended for testing purpose and 7-11 for production by LLVM. +Memory and slowdown numbers are only reported for LLVM sanitizers. Zig does not +report own numbers yet (2025-01-11). Slowdown for dynamic sanitizer versions +increases by a factor of 10x in contrast to the listed static usage costs.. +The leak sanitizer does only check for memory leaks, not other system resources. +Besides various Kernel specific tools to track system resources, +Valgrind can be used on Posix systems for non-memory resources and +Application Verifier for Windows. +Address and thread sanitizers can not be combined in Clang and combined usage +of the Zig implementation is limited by virtual memory usage. +In Zig, aliasing can currently not be sanitized against whereas in Clang only +typed based aliasing can be sanitized without any numbers reported by LLVM yet. + +Address and thread sanitizers can not be combined. +* recommended -O1 -g -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-common +* overview https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html * docs https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer * extras outside of -fsanitize=address use -O1 or higher, -fno-omit-frame-pointer, no TCE -fno-optimize-sibling-calls -Typical slowdown 2x ASAN_OPTIONS=detect_stack_use_after_return=1/0 ASAN_OPTIONS=check_initialization_order=1 (not supported on macOS) ASAN_OPTIONS=detect_leaks=1 (no support on windows) -* extras outside of -fsanitize=leak -minimal performance and memory overhead until end of process (leak checking phase) +* extras Leak sanitizer (-fsanitize=leak) also available configurably as ASAN option: ASAN_OPTIONS=detect_leaks=1 -* extras outside of incompat to address -fsanitize=memory -Uninitialized Memory sanitizer +* extras Uninitialized Memory sanitizer (-fsanitize=memory) use -O1 or higher, -fno-omit-frame-pointer, no TCE -fno-optimize-sibling-calls Memory usage 2x to 3x Typical slowdown 3x -* extras outside of -fsanitize=undefined +* extras Undefined Behavior sanitizer (-fsanitize=undefined) Extra options -fsanitize=unsigned-integer-overflow -fsanitize=implicit-conversion -fsanitize=array-bounds -fsanitize=nullability -* extras outside of -fsanitize=thread +* extras Thread sanitizer -fsanitize=thread use -O1 or higher -g for file names and line numbers -Memory usage 5x to 10x + 1MB per thread -Typical slowdown 5x to 15x * extras outside of -fsanitize=type -TODO +TODO no data and docs yet 6. maintain debug infos in separate files objcopy --only-keep-debug executable_file executable_file.debug diff --git a/templates/common.cpp b/templates/common.cpp index 5d1560d..540a711 100644 --- a/templates/common.cpp +++ b/templates/common.cpp @@ -1593,9 +1593,11 @@ void ape_printing_bad() { void ape_print(); void ape_print() { FILE *f1 = fopen("file1", "a+"); - fprintf(f1, "sometext\n"); - fprintf(f1, "f1 ptr: %p\n", static_cast(f1)); - fclose(f1); + if (f1 != nullptr) { + fprintf(f1, "sometext\n"); + fprintf(f1, "f1 ptr: %p\n", static_cast(f1)); + fclose(f1); + } } [[noreturn]] void ape_throw(); diff --git a/templates/common.css b/templates/common.css index abf8ebc..a723d7a 100644 --- a/templates/common.css +++ b/templates/common.css @@ -99,3 +99,27 @@ .home_left { flex: 1; width: 300px; } .home_right { flex: 1; width: 300px; } @media screen and (max-width: 600px) { .home_page { display: block; } } + +/* table with auto layout */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/* */ +/*
EdWood
AlbertSchweitzer
*/ +/* table { */ +/* table-layout: auto; */ +/* width: 120px; */ +/* border: 1px solid red; */ +/* } */ +/**/ +/* td { */ +/* border: 1px solid blue; */ +/* overflow: hidden; */ +/* white-space: nowrap; */ +/* text-overflow: ellipsis; */ +/* } */ diff --git a/templates/common.tex b/templates/common.tex index f5c20f1..de4bf7f 100644 --- a/templates/common.tex +++ b/templates/common.tex @@ -1,6 +1,7 @@ % TODO add my standard tooling and commands % https://github.com/WGUNDERWOOD/tex-fmt +% https://github.com/mcopik/latex-tools % tex-fmt: off \textrm{} \textrm{roman} text Roman text @@ -15,3 +16,7 @@ \emph{} \emph{emphasized} text emphasized text \textnormal{} \textnormal{default} text default text % tex-fmt: on + +SHENNANIGAN +# in href needs to be escaped +Explained \href{https://github.com/ziglang/zig/issues/6152\#issuecomment-1369286411}{\color{darkblue}runtime methods to detect resources leakages} including trade-offs. diff --git a/templates/cpu b/templates/cpu index 3b8b2ff..ddbeeb8 100644 --- a/templates/cpu +++ b/templates/cpu @@ -127,6 +127,9 @@ effects? #### Segmentation CPU feature for sub-page access granularity TODO go through and test guide https://nixhacker.com/segmentation-in-intel-64-bit/ +* amd64 typically use a vestigial feature that used to be used for segmentation for tls +but it's not segmentation anymore +* basically just an extra register whose assigned role is 'always hold the tls base' -#### file_sytems -TODO wait for it https://blog.jandroegehoff.de/ \ No newline at end of file +#### file_systems +TODO wait for it https://blog.jandroegehoff.de/ diff --git a/templates/formal_modeling.txt b/templates/formal_modeling.txt index bd6dfe8..c48bf07 100644 --- a/templates/formal_modeling.txt +++ b/templates/formal_modeling.txt @@ -417,6 +417,13 @@ https://buttondown.email/hillelwayne/archive/nondeterminism-in-formal-specificat self = fieldParentPtr("task", task) if self.cancelled.load(): return // actual work +- 24. robust reachability as "perfect reproducibility" of violations controlled by attacker + * Inference of Robust Reachability Constraints by Sellami et al. + * paper works via satisfiability checking; looks fairly minimal + * taint analysis does this on input data, might be possible to do coverage based fuzzing; however not minimal +- 25. static analysis to prove bug-free programs + * https://github.com/codex-semantics-library/codex not usable yet + * https://github.com/binsec/binsec unclear trade-offs of symbolic execution To me unclear if solved: * check if linker speak has a formal model diff --git a/templates/gpu b/templates/gpu index b52ef39..e131bdf 100644 --- a/templates/gpu +++ b/templates/gpu @@ -14,3 +14,12 @@ https://github.com/nDimensional/andromeda effortless CPU + GPU programming https://chapel-lang.org/ + +2D graphics +Skia pipeline undocumented +most likely complex path+color operations are the latency adders, everything else should map to GPU semantics (if possible) +https://stackoverflow.com/questions/5762727/how-do-the-pieces-of-androids-2d-canvas-drawing-pipeline-fit-together +https://skia.org/docs/user/sksl/ + +GPU pipeline +vertex specification -> vertex shader -> tessellation -> geometry shader -> vertex post-processing -> primitive assembly -> rasterization -> fragment shader -> per-sample operation diff --git a/templates/kernel_semantics b/templates/kernel_semantics index a32b789..ad08279 100644 --- a/templates/kernel_semantics +++ b/templates/kernel_semantics @@ -1,5 +1,5 @@ Overview of Kernel use case groups -- excution environments (kernel [task, module, virtual machine], user [process, thread]) +- eexcution environments (kernel [task, module, virtual machine], user [process, thread]) - inter process communication (pipe, io_uring?, locks, shared memory, signals, sockets, streams, wait groups, ..) - io management (block, io_uring, stream, virtual device access, ..) - memory management (kernel, stack, heap, paging [if CPU has MMU, PMU], ..) @@ -35,7 +35,7 @@ List of fundamental and (in practice) unfixable design flaws * (non)cooperative scheduling could use suspension points + Kernel watchdog with suspension points being necessary async events and watchdog being updated on every scheduler iteration - + leaks (hardware) time informations or requires configurable timesteps over it + + leaks (hardware) time information or requires configurable timesteps over it - Unix? file permission system stores information within each file instead of file group * nested overlayfs has limit 6 blocking usage to make nix on Linux less path hacky - Linux/Unix clone forces copy of whole memory, file descriptors etc (needed on @@ -46,3 +46,6 @@ List of fundamental and (in practice) unfixable design flaws ABI stable idea citations for everything + +mini kernel +https://github.com/nuta/operating-system-in-1000-lines diff --git a/templates/parallel_programming.txt b/templates/parallel_programming.txt index edffba3..314b414 100644 --- a/templates/parallel_programming.txt +++ b/templates/parallel_programming.txt @@ -1,9 +1,9 @@ see for impls and techniques ./example/parallel_programming/README.md Based on talk by kprotty "Proficient Parallel Programming" -paralellism: doing 2 or more things at the same time +parallelism: doing 2 or more things at the same time concurrency: doing 2 things not necessary at the same time -OS provides paralellism on top of concurrency. +OS provides parallelism on top of concurrency. Why? - Getting things done faster @@ -26,7 +26,7 @@ Use cases - more computation Dilemma -- single threaded and determinstic (best for fuzzing and simulation testing) +- single threaded and deterministic (best for fuzzing and simulation testing) - trace of program run as flamegraph * shows major blocking parts * problem: sorting => solution: faster, incremental @@ -70,13 +70,13 @@ Solution Change to chained wake Competition Run as many tasks as you can Race to register pattern (once, CPU pipelining): -- doing work nontheless and the first one is allowed to write +- doing work nonetheless and the first one is allowed to write - minimize time in critical section - avoid (deferred) handoffs * good for fairness * bad for throughput - chennal > mutex, because its easier to mess up size of critical section -- WaitGroup instad of multiple join(): do 1 syscall instead of N syscalls +- WaitGroup instead of multiple join(): do 1 syscall instead of N syscalls - EventListener > ConditionVariable: wait for arbitrary condition without walk through mutex - Channels too much overhead: Prefer RwLock * instead of Mutex: Writers block everyone, Readers only block Writers @@ -95,13 +95,13 @@ while true: ``` - for simple instructions use the hardware primitives ``` -old = atmoic_fetch_add(ptr, 1) +old = atomic_fetch_add(ptr, 1) ``` - too complex operation * use backoff operation for ca. 2x throughput ``` while true: - old = atmoic_load(ptr) + old = atomic_load(ptr) new = complex(old) _ = aomitc_cas(ptr, old, new) catch break backoff() @@ -149,7 +149,7 @@ ABA_problem General * best class of algorithms is wait free (meaning waiting an as low as possible and bounded time, usually quadratic to number of threads) * all recent high performant algorithms use hazard pointers as vector/array of [0,MAX_THREAD_ID] -* getting the size, if the storage is non-continous, has a perf cost +* getting the size, if the storage is non-continuous, has a perf cost * reliably detect, if a pop emptied a data storage, on the same thread after the pop, has a perf cost * vice versa detecting that the storage ran full * most papers have no reference implementation with tests linked or can have @@ -179,4 +179,4 @@ Simple * https://github.com/rigtorp/MPMCQueue bounded multi-producer multi-consumer concurrent queue written in C++11 Erik Rigtorp * "Simple, Fast, and Practical Non-Blocking and Blocking Concurrent Queue Algorithms" by Maged M. Michael and Michael L. Scott. https://github.com/Qarterd/Honeycomb/blob/master/src/common/Honey/Thread/LockFree/Queue.h - https://www.cs.rochester.edu/research/synchronization/pseudocode/queues.html \ No newline at end of file + https://www.cs.rochester.edu/research/synchronization/pseudocode/queues.html diff --git a/templates/security b/templates/security index 6ec5fec..e4434ca 100644 --- a/templates/security +++ b/templates/security @@ -110,6 +110,8 @@ which means the tracee must be in a sub-domain of the tracer." Unfortunately it does not say "must have". 7. disable hyperthreading (in bios) 8. hardened kernel? +9. good practical usage https://blog.emilua.org/2025/01/12/software-sandboxing-basics/ + * no comparison to Arcan yet Windows "Win32 app isolation" TODO list capabilities diff --git a/templates/testing b/templates/testing index f2d644a..aa6787c 100644 --- a/templates/testing +++ b/templates/testing @@ -2,7 +2,6 @@ https://lwn.net/Articles/995276/#Comments todo summarize https://mutants.rs/ - Best Practice taken from "Large Scale Architecture: The Unreasonable Effectiveness of Simplicity - Randy Shoup" and extended by personal experience. Related and partially very funny https://grugbrain.dev/ @@ -203,4 +202,8 @@ https://antithesis.com/docs/best_practices/sizing.html TODO best practice https://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html -fuzzing ideas + process semantics \ No newline at end of file +fuzzing ideas + process semantics + +https://skia.org/docs/dev/testing/automated_testing/ +https://github.com/luci/luci-py/blob/main/appengine/swarming/doc/Design.md +https://github.com/luci/luci-py/blob/main/appengine/swarming/doc/Detailed-Design.md diff --git a/templates/tooling b/templates/tooling index 0942b25..d19b7ba 100644 --- a/templates/tooling +++ b/templates/tooling @@ -130,6 +130,11 @@ perf trace record perf script perf report +linux perf, windows etw, macos kperf c api usage +https://gist.github.com/mmozeiko/bd5923bcd9d20b5b9946691932ec95fa?ts=4 +macos only, no cli and gui yet +https://gist.github.com/ibireme/173517c208c7dc333ba962c1f0d67d12 + gdb record btrace