From 6975dad87b6c2e4302191b2c8f6bff119a0089ed Mon Sep 17 00:00:00 2001 From: nntrn <17685332+nntrn@users.noreply.github.com> Date: Thu, 14 Dec 2023 03:23:53 -0600 Subject: [PATCH] Add gray matter --- data/unroll-example.json | 38 ++++++++ functions/barcharts.md | 4 +- functions/conversion.md | 4 +- functions/describe.md | 5 +- functions/flatten.md | 6 +- functions/github-api.md | 4 +- functions/json2csv.md | 4 +- functions/pick.md | 4 +- functions/read-history.md | 4 +- functions/summary.md | 4 +- functions/unroll.md | 94 +++++++++++++++++++ general/codepoints.md | 4 +- general/data-cleaning.md | 4 +- .../data-transform.md | 7 +- general/examples.md | 4 +- general/inputs.md | 5 +- general/kv.md | 4 +- general/oneliners.md | 4 +- general/reduce.md | 4 +- general/scalars.md | 4 +- general/streams.md | 4 +- general/wrangle.md | 4 +- readme.md | 10 ++ unroll.jq | 62 ++++++++++++ 24 files changed, 270 insertions(+), 21 deletions(-) create mode 100644 data/unroll-example.json create mode 100644 functions/unroll.md rename data-transform.md => general/data-transform.md (90%) create mode 100755 unroll.jq diff --git a/data/unroll-example.json b/data/unroll-example.json new file mode 100644 index 0000000..9f34d88 --- /dev/null +++ b/data/unroll-example.json @@ -0,0 +1,38 @@ +{ + "results": [ + { + "id": 306, + "name": "First Company", + "branches": [ + { + "id": 4191, + "city": "Seattle", + "customers": [ + { + "id": 446, + "name": "Big Tech 1" + }, + { + "id": 447, + "name": "Big Tech 2" + } + ] + }, + { + "id": 4192, + "city": "Oakland", + "customers": [ + { + "id": 448, + "name": "Health Tech 1" + }, + { + "id": 449, + "name": "Health Tech 2" + } + ] + } + ] + } + ] +} diff --git a/functions/barcharts.md b/functions/barcharts.md index bb31328..655a065 100644 --- a/functions/barcharts.md +++ b/functions/barcharts.md @@ -1,4 +1,6 @@ -# barcharts +--- +title: barcharts +--- ```jq def barchart($key): diff --git a/functions/conversion.md b/functions/conversion.md index b92bf5f..656886a 100644 --- a/functions/conversion.md +++ b/functions/conversion.md @@ -1,4 +1,6 @@ -# conversion +--- +title: conversion +--- Pretty print numbers diff --git a/functions/describe.md b/functions/describe.md index 15bf474..aefc3fa 100644 --- a/functions/describe.md +++ b/functions/describe.md @@ -1,4 +1,7 @@ -# describe +--- +title: describe +data: data/sample-github-events.json +--- Get object outline diff --git a/functions/flatten.md b/functions/flatten.md index 45be442..bf52be2 100644 --- a/functions/flatten.md +++ b/functions/flatten.md @@ -1,4 +1,8 @@ -# Flatten +--- +title: Flatten +data: data/sample-github-events.json +--- + ## Flat JSON diff --git a/functions/github-api.md b/functions/github-api.md index ecb89bc..8996c2c 100644 --- a/functions/github-api.md +++ b/functions/github-api.md @@ -1,4 +1,6 @@ -# github api +--- +title: github api +--- ```jq def github_raw_url: diff --git a/functions/json2csv.md b/functions/json2csv.md index 04f2732..656cc3d 100644 --- a/functions/json2csv.md +++ b/functions/json2csv.md @@ -1,4 +1,6 @@ -# json2csv +--- +title: json2csv +--- ```jq # ~/.jq/convert.jq diff --git a/functions/pick.md b/functions/pick.md index dbd31a4..2186566 100644 --- a/functions/pick.md +++ b/functions/pick.md @@ -1,4 +1,6 @@ -# pick +--- +title: pick +--- [[Source]](https://github.com/jqlang/jq/issues/2578#issuecomment-1532632453) diff --git a/functions/read-history.md b/functions/read-history.md index a622df5..a56b318 100644 --- a/functions/read-history.md +++ b/functions/read-history.md @@ -1,4 +1,6 @@ -# Read history +--- +title: Read history +--- Compile timestamps in `.bash_history` for all users diff --git a/functions/summary.md b/functions/summary.md index ca73947..fa8b330 100644 --- a/functions/summary.md +++ b/functions/summary.md @@ -1,4 +1,6 @@ -# summary +--- +title: summary +--- ```jq def grouped_summary($item): diff --git a/functions/unroll.md b/functions/unroll.md new file mode 100644 index 0000000..d337b8c --- /dev/null +++ b/functions/unroll.md @@ -0,0 +1,94 @@ +--- +title: Unroll +source: + - https://stackoverflow.com/a/33290267 +--- + + +```jq +[leaf_paths as $path | { + "key": $path | map(tostring) | join("_"), + "value": getpath($path) +}] | from_entries +``` + +```console +$ cat data/nested.json|jq '[leaf_paths as $path | { + "key": $path | map(tostring) | join("_"), + "value": getpath($path) +}] | from_entries +' +{ + "server_atx_user": "annie", + "server_atx_port": 22, + "storage_nyc_user": "nntrn", + "storage_nyc_port": 22 +} +``` + + + +```jq +def categorize: + # Returns "object", "array" or "scalar" to indicate the category + # of the piped element. + if type == "object" then "object" + elif type == "array" then "array" + else "scalar" + end; + +def pluck($category): + # Plucks the children of a particular category from piped element. + if categorize != "object" + then empty + else to_entries[] + | select(.value | categorize == $category) + | .value + end; + +def split: + # Splits the piped element's children into arrays, scalars, and objects + # and returns a meta object containing the children seperated by these + # keys. If the piped element is a scalar or array, this does not look + # at the children, but just returns that element in the meta object. + if categorize == "scalar" then { objects: [], arrays: [], scalars: [.] } + elif categorize == "array" then { objects: [], arrays: [.], scalars: [] } + else { objects: [pluck("object")], arrays : [pluck("array")], scalars: [pluck("scalar")] } + end; + +def unwrap: + # Unwraps an array recursively, until the elements of the returned array + # are either scalars or objects but not arrays. If piped element is not + # an array, returns the element as is. + + if type != "array" then . + elif length == 0 then empty + else .[] | unwrap + end; + +def extract($category): + # Extracts the elements of a particular category from the piped in array. + # If the piped in element is not an array, this fn acts as filter to + # only return the element if it is of the desired category. + unwrap | select(.| categorize == $category); + +def unroll: + # Unrolls the passed in object recursively until only scalars are left. + # Returns a row for each leaf node of tree structure of the object and + # elements of the row would be all the scalars encountered at all the + # ancestor levels of this left node. + + . | .result += .state.scalars + | .state.objects += [.state.arrays | extract("object")] + | .state.objects += [.state.arrays | extract("scalar")] + | if (.state.objects | length == 0 ) + then .result + else ({ data : .state.objects, + state: .state.objects[] | split, + result: .result + } | unroll) + end; + +def unrolls($data): { data: $data, state: $data| split, result: [] } | unroll ; +def unrolls: unrolls(.); +``` diff --git a/general/codepoints.md b/general/codepoints.md index edf1321..08f1de3 100644 --- a/general/codepoints.md +++ b/general/codepoints.md @@ -1,4 +1,6 @@ -# Codepoints +--- +title: Codepoints +--- Work with explode/implode diff --git a/general/data-cleaning.md b/general/data-cleaning.md index 0ce1780..496160e 100644 --- a/general/data-cleaning.md +++ b/general/data-cleaning.md @@ -1,4 +1,6 @@ -# Data cleaning +--- +title: Data cleaning +--- ## Delete keys diff --git a/data-transform.md b/general/data-transform.md similarity index 90% rename from data-transform.md rename to general/data-transform.md index f6944df..088d611 100644 --- a/data-transform.md +++ b/general/data-transform.md @@ -1,10 +1,13 @@ -# Data transformation +--- +title: Data transformation +data: data/nested.json +--- ## json2ini Adapted from https://stackoverflow.com/a/76665197 -Data used: [nested.json](data/nested.json) +Data used: [nested.json](../data/nested.json) ``` { diff --git a/general/examples.md b/general/examples.md index b40f98f..3b671d4 100644 --- a/general/examples.md +++ b/general/examples.md @@ -1,4 +1,6 @@ -# Examples +--- +title: Examples +--- ## Reduce diff --git a/general/inputs.md b/general/inputs.md index 443285d..3325350 100644 --- a/general/inputs.md +++ b/general/inputs.md @@ -1,4 +1,7 @@ -# Working with `inputs` +--- +title: Inputs +data: data/tmp.tsv +--- **DATA**: [tmp.tsv](../data/tmp.tsv) diff --git a/general/kv.md b/general/kv.md index b1a112e..3dc47bc 100644 --- a/general/kv.md +++ b/general/kv.md @@ -1,4 +1,6 @@ -# kv +--- +title: kv +--- ```jq split("\n") diff --git a/general/oneliners.md b/general/oneliners.md index b2c8fcf..1261356 100644 --- a/general/oneliners.md +++ b/general/oneliners.md @@ -1,4 +1,6 @@ -# Oneliners +--- +title: Oneliners +--- ## Recursively convert appropriate data types from string diff --git a/general/reduce.md b/general/reduce.md index ac64882..c425f81 100644 --- a/general/reduce.md +++ b/general/reduce.md @@ -1,4 +1,6 @@ -# Reduce +--- +title: Reduce +--- ```jq def tocsv: diff --git a/general/scalars.md b/general/scalars.md index 52ed626..2e09ad0 100644 --- a/general/scalars.md +++ b/general/scalars.md @@ -1,4 +1,6 @@ -# Scalars +--- +title: Scalars +--- Scalars are variables that hold an *individual* value (strings, integers, and booleans). If it's not an object or array — it's most likely a scalar diff --git a/general/streams.md b/general/streams.md index 61e0513..a46d1e9 100644 --- a/general/streams.md +++ b/general/streams.md @@ -1,4 +1,6 @@ -# Working with streams +--- +title: Working with streams +--- Data used: [nested.json](../data/outer.json) diff --git a/general/wrangle.md b/general/wrangle.md index 3cd2a51..733e03a 100644 --- a/general/wrangle.md +++ b/general/wrangle.md @@ -1,4 +1,6 @@ -# Wrangle +--- +title: Wrangle +--- ## Zip Column Headers diff --git a/readme.md b/readme.md index 81d0b9e..ae650f6 100644 --- a/readme.md +++ b/readme.md @@ -6,4 +6,14 @@ git clone -b staging https://github.com/nntrn/jq-recipes.git git clone https://github.com/nntrn/jq-recipes.wiki.git ``` +## Setup + +```sh +curl --create-dirs -o ~/.jq/recipes.jq https://nntrn.github.io/jq-recipes/recipes.jq + +git clone -b staging https://github.com/nntrn/jq-recipes.git + + +``` + [Github Page](http://nntrn.github.io/jq-recipes) | [Wiki](https://github.com/nntrn/jq-recipes/wiki) diff --git a/unroll.jq b/unroll.jq new file mode 100755 index 0000000..7d48b9f --- /dev/null +++ b/unroll.jq @@ -0,0 +1,62 @@ +def categorize: + # Returns "object", "array" or "scalar" to indicate the category + # of the piped element. + if type == "object" then "object" + elif type == "array" then "array" + else "scalar" + end; + +def pluck($category): + # Plucks the children of a particular category from piped element. + if categorize != "object" + then empty + else to_entries[] + | select(.value | categorize == $category) + | .value + end; + +def split: + # Splits the piped element's children into arrays, scalars, and objects + # and returns a meta object containing the children seperated by these + # keys. If the piped element is a scalar or array, this does not look + # at the children, but just returns that element in the meta object. + if categorize == "scalar" then { objects: [], arrays: [], scalars: [.] } + elif categorize == "array" then { objects: [], arrays: [.], scalars: [] } + else { objects: [pluck("object")], arrays : [pluck("array")], scalars: [pluck("scalar")] } + end; + +def unwrap: + # Unwraps an array recursively, until the elements of the returned array + # are either scalars or objects but not arrays. If piped element is not + # an array, returns the element as is. + + if type != "array" then . + elif length == 0 then empty + else .[] | unwrap + end; + +def extract($category): + # Extracts the elements of a particular category from the piped in array. + # If the piped in element is not an array, this fn acts as filter to + # only return the element if it is of the desired category. + unwrap | select(.| categorize == $category); + +def unroll: + # Unrolls the passed in object recursively until only scalars are left. + # Returns a row for each leaf node of tree structure of the object and + # elements of the row would be all the scalars encountered at all the + # ancestor levels of this left node. + + . | .result += .state.scalars + | .state.objects += [.state.arrays | extract("object")] + | .state.objects += [.state.arrays | extract("scalar")] + | if (.state.objects | length == 0 ) + then .result + else ({ data : .state.objects, + state: .state.objects[] | split, + result: .result + } | unroll) + end; + +def unrolls($data): { data: $data, state: $data| split, result: [] } | unroll ; +def unrolls: unrolls(.);