From 48b0facc76a926cce1f569ca9ba33ed76c03e196 Mon Sep 17 00:00:00 2001 From: Gabe Venberg Date: Sat, 9 Mar 2024 09:57:00 -0600 Subject: [PATCH] halfway done with the nushell post. --- content/posts/nushell.adoc | 97 ---------------------- content/posts/nushell.md | 164 +++++++++++++++++++++++++++++++++++++ hugo.toml | 4 +- 3 files changed, 166 insertions(+), 99 deletions(-) delete mode 100644 content/posts/nushell.adoc create mode 100644 content/posts/nushell.md diff --git a/content/posts/nushell.adoc b/content/posts/nushell.adoc deleted file mode 100644 index 471ae74..0000000 --- a/content/posts/nushell.adoc +++ /dev/null @@ -1,97 +0,0 @@ -+++ -title = "Nushell first impressions" -date = 2024-03-01T11:34:04-06:00 -draft = true -+++ -:caution-caption: pass:[☠] -:important-caption: pass:[❗] -:note-caption: pass:[✏️] -:tip-caption: pass:[💡] -:warning-caption: pass:[⚠] -:toc: -:toclevels: 6 - -Ive been trying out a bunch of new shell utilites lately, -switching up my shell, terminal multiplexer, and even experimenting with my editor. -Today, Id like to focus on my experiments with my shell. - -== My old setup - -Before this, I had been using a minimal zsh setup for a long time, -with only built in features and a handmade prompt. -Zsh is a good shell, probably one of the best posix shells out there, -and I still use it when a posix shell is needed. - -However, I got tired of the endless footguns that posix shell scripting imposes, -easy to make errors around quoting, word splitting, and escaping, -the sort of thing that makes https://www.shellcheck.net/[shellcheck] necessary. - -I played around with fish for a few days, -but it had many of the same fundamental design choices, mainly, being 'stringly typed', -that made posix shells such a displeasure to work with. - -== A Nu shell - -While googling around for alternative shells, I stumbled across https://www.nushell.sh/[nushell], -a shell that claimed to work around structured data instead of just strings. -This was *exactly* what I was looking for, and I installed it immediately. -I decided to work with it for around a month, -give myself enough time to really use it, -see not only how it felt with ordinary usage, -but to give myself time and opportunity to construct a few pipelines and scripts in it. - -All that said, the month is up, and ive been collecting examples, -thoughts, and some criticisms along the way. - -== Piping structured data - -// show some examples of grouping, sorting, etc without endless invocations of `cut`. - -== Parsing non-nu tools - -// show parsing initcall_debug logs, and how it then lets one do analysis on it -``` -[ 0.518096] calling prandom_reseed+0x0/0x40 @ 1 -[ 0.518119] initcall prandom_reseed+0x0/0x40 returned 0 after 12 usecs -[ 0.518127] calling clk_debug_init+0x0/0x140 @ 1 -[ 0.531128] initcall clk_debug_init+0x0/0x140 returned 0 after 12659 usecs -[ 0.531145] calling imx_amp_power_init+0x0/0x13c @ 1 -[ 0.531156] initcall imx_amp_power_init+0x0/0x13c returned 0 after 1 usecs -[ 0.531175] calling deferred_probe_initcall+0x0/0x3c @ 1 -[ 0.532275] imx_thermal 2000000.aips-bus:tempmon: Automotive CPU temperature grade - max:125C critical:120C passive:115C -[ 0.533166] pcm1753 pcm1753: Setting maximum volume limit of 226 -[ 0.535167] asoc-simple-card sound: PCM1753-HiFi <-> 2024000.esai mapping ok -[ 0.536199] initcall deferred_probe_initcall+0x0/0x3c returned 0 after 4880 usecs -[ 0.536217] calling pm_genpd_debug_init+0x0/0x70 @ 1 -[ 0.536249] initcall pm_genpd_debug_init+0x0/0x70 returned 0 after 20 usecs -[ 0.536262] calling genpd_poweroff_unused+0x0/0x8c @ 1 -[ 0.536284] initcall genpd_poweroff_unused+0x0/0x8c returned 0 after 11 usecs -[ 0.536296] calling gpio_keys_init+0x0/0x20 @ 1 -``` - -{{}} -open $file | -lines | -find '] initcall ' | -parse -r '\[\s*(?\d+\.\d+)\] (?.+) returned (?\d+) after (?.+)' | -update timestamp {into float} | -update delta {str replace ' usecs' 'us'} | -update delta {into duration} | -update return {into int} | -move delta --after timestamp -{{}} - -== Defining custom commands - -// show the basic syntax for custom commands - -=== Built in arg parsing? - -// show syntax for custom args, and how it leads to auto completion and help generation. - -== Error messages - -== Whats not there yet - -// explain some limitations, tools that assume the existence of a posix shell (esp files one is instructed to source) -// also explain the limitations where nushell scripts cannot pass structured data, but are treated as external commands, therefore their usefullness in a pipeline is limited. diff --git a/content/posts/nushell.md b/content/posts/nushell.md new file mode 100644 index 0000000..40db913 --- /dev/null +++ b/content/posts/nushell.md @@ -0,0 +1,164 @@ ++++ +title = "Nushell first impressions" +date = 2024-03-01T11:34:04-06:00 +draft = true ++++ + +Ive been trying out a bunch of new shell utilities lately, +switching up my shell, terminal multiplexer, and even experimenting with my editor. +Today, Id like to focus on my experiments with my shell. + +## My old setup + +Before this, I had been using a minimal zsh setup for a long time, +with only built in features and a handmade prompt. +Zsh is a good shell, probably one of the best posix shells out there, +and I still use it when a posix shell is needed. + +However, I got tired of the endless footguns that posix shell scripting imposes, +easy to make errors around quoting, word splitting, and escaping, +the sort of thing that makes [shellcheck](https://www.shellcheck.net/) necessary. + +I played around with fish for a few days, +but it had many of the same fundamental design choices, mainly, being 'stringly typed', +that made posix shells such a displeasure to work with. + +## A Nu shell + +While googling around for alternative shells, I stumbled across [nushell](https://www.nushell.sh/), +a shell that claimed to work around structured data instead of just strings. +This was *exactly* what I was looking for, and I installed it immediately. +I decided to work with it for around a month, +give myself enough time to really use it, +see not only how it felt with ordinary usage, +but to give myself time and opportunity to construct a few pipelines and scripts in it. + +All that said, the month is up, and Ive been collecting examples, +thoughts, and some criticisms along the way. + +## Piping structured data + +One of the core features of nushell is that commands return structured data, +instead of plain strings. +Pipelines can pass lists, records, or tables. +Individual entries can be one of several built in datatypes, +including rich datatypes like datetimes, durations, and filesizes. + +Nushell can also open many filetypes and turn them into nushell native datastructures to work with, +including csv's, json, toml, yaml, xml, sqlite files, and even excel and libreoffice calc spreadsheets. + +Once you have your data in nushell datastructures, +you can do all sorts of manipulations on it. +It feels like an interesting mix of functional programming and SQL, +but it actually works really, really well. +You can sort, filter, and aggregate the data, +use a SQL style join statement between two tables, +and use functional programming patterns to manipulate tables. + +Some examples of things that nushell enables with this structured data passing +through pipelines includes: + +{{}} +# show all files recurisively that were modified in the last week +ls **/* | where modified > ( + # create timestamp from relative human readable string. + '1 week ago' | into datetime +) +{{}} + +{{}} +# show all executables in the current directory that are currently running. +ps | +# convert the name of the called process into a path +update name {|process| (which $process.name).path.0?} | +# join with the list of all files in the current directory, recursing down subdirectories. +join (ls -f **/*) name +{{}} + +{{}} +# show all values in 1 csv but not in another +open all_tasks.csv | +# filter out tasks that cause the closure to return false +filter { |task| + not ( + # check if the task number is in the other csv. + $task.number in ( + open tasks_done.csv | + get 'task_number' + ) + ) +} +{{}} + +All of these can be one liners, but have been broken up in order to insert +explanatory comments. + +## Parsing non-nu tools + +But what if our tool/text file isnt in a format nushell understands? +Thankfully, for most formats, parsing is relatively straightforward. +Lets take this NGINX server log, for example (not a log of real traffic, just a +sample log I found) + +{{}} +135.125.217.54 - - [27/Mar/2023:12:57:44 +0000] "GET /.env HTTP/1.1" 404 197 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" +135.125.217.54 - - [27/Mar/2023:12:57:44 +0000] "POST / HTTP/1.1" 405 568 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" +43.154.141.71 - - [27/Mar/2023:12:58:04 +0000] "HEAD /Core/Skin/Login.aspx HTTP/1.1" 404 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36" +193.32.162.159 - - [27/Mar/2023:13:01:07 +0000] "GET / HTTP/1.1" 200 13 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46" +193.32.162.159 - - [27/Mar/2023:13:01:18 +0000] "GET /dispatch.asp HTTP/1.1" 404 197 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36 Edg/90.0.818.46" + +{{}} + +We can parse it into a nu table like so (each line has a comment explaining what +it does, for those unfamiliar with the nushell language): + +{{}} +open access.log | +# turn into a list of lines +lines | +# parse into a table +parse '{ip} - {user} [{time}] "{request_type} {request}" {status} {bytes_sent} "{referrer}" "{user_agent}"' | +# parse time into proper datetime +update time {into datetime -f '%d/%b/%Y:%T %z'} | +# parse into proper integer +update bytes_sent {into int} +{{}} + +Now that we have it in nushell tables, we can bring all of nushells tools to +bear on the data. We could, for example, plot a histogram of the most common +ips, just by piping the whole thing into `histogram ip`. We could easily +calculate the average bytes sent per request. We could group the records by the +day or hour they happened, and analyze each of those groups independently. And +we can do all of that after arbitrarily filtering, sorting, or otherwise +transforming the table. + +While it would be a pretty long one liner if we decided to put it in a single +line, its still quite easy and straightforward to write. +Most log formats and command outputs are similarly straightforward. + +## Defining custom commands + +// show the basic syntax for custom commands + +### Built in arg parsing? + +// show syntax for custom args, and how it leads to auto completion and help generation. + +## Error messages + +## Whats not there yet + +Now, nushell is not finished yet. +As I write, I am running version 0.91 of nu. +Similar to fish, it not being a POSIX shell means that you still need to drop +into bash or zsh in order to source env files in order to, +for example, use a cross-compiling c/c++ sdk. +(thankfully, python virtualenvs already come with a nu script for you to source, +so doing python dev will not require you to launch a POSIX shell) + +Additionally, while you can write nu script files, +invoking them from within nu treats them as external commands, +meaning they take in and pass out plain text, +rather than the structured data that you would get with a proper custom command +or nu builtin. +//explain the best workaround. diff --git a/hugo.toml b/hugo.toml index e14403b..8ec1edb 100644 --- a/hugo.toml +++ b/hugo.toml @@ -6,7 +6,7 @@ enableRobotsTXT = true [params] env = 'production' title = 'Mildly interesting' -description = "Mildly interesting Things ive Done" +description = "Mildly Interesting Writings" author = 'Gabe Venberg' DateFormat = "2006-1-2" defaultTheme = 'dark' @@ -35,7 +35,7 @@ disableHLJS = true [params.homeInfoParams] Title = "Mildly Interesting" Content = """ -Mildly interesting things Ive done. +Mildly interesting writings. """ [[params.socialIcons]]