From 869b4c149e8b0996f292a2a76e27ebb3f09cc463 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sat, 6 Jul 2024 16:26:04 +0100 Subject: [PATCH 1/6] Add tmf-pipeline's data visualisation --- src/lib/server/build.ml | 91 +++++++++ src/lib/server/dune | 2 +- src/lib/server/pages.ml | 342 +++++++++++++++++++++++++++++++++ src/lib/server/shark_server.ml | 3 + vendor/obuilder | 2 +- 5 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 src/lib/server/build.ml create mode 100644 src/lib/server/pages.ml diff --git a/src/lib/server/build.ml b/src/lib/server/build.ml new file mode 100644 index 00000000..df7640f9 --- /dev/null +++ b/src/lib/server/build.ml @@ -0,0 +1,91 @@ +let ( / ) = Filename.concat + +let find_paths pred paths = + let rec loop acc = function + | `Dir (_, more) -> List.fold_left loop acc more + | `File (path, _) -> if pred path then path :: acc else acc + | _ -> acc + in + loop [] paths + +(* Rendering Builds *) +let render (Obuilder.Store_spec.Store ((module Store), store)) id = + let result = Lwt_eio.run_lwt (fun () -> Store.result store id) in + match result with + | None -> + Cohttp_eio.Server.respond_string ~status:`OK ~body:("No result for " ^ id) + () + | Some path -> + let manifest, geojsons, jsons, images, tabular = + let src_dir = Fpath.(v path / "rootfs") |> Fpath.to_string in + match Obuilder.Manifest.generate ~exclude:[] ~src_dir "data" with + | Error (`Msg m) -> (m, [], [], [], []) + | Ok src_manifest -> + let geojsons = + find_paths + (fun p -> Filename.extension p = ".geojson") + src_manifest + in + let jsons = + find_paths (fun p -> Filename.extension p = ".json") src_manifest + in + let geojsons = + List.map Uri.pct_encode geojsons + |> List.map (fun v -> + "." / id + / (Uri.with_query (Uri.of_string "serve") + [ ("file", [ v ]) ] + |> Uri.to_string)) + in + let jsons = + List.map + (fun v -> + In_channel.with_open_bin (Filename.concat src_dir v) + @@ In_channel.input_all) + jsons + in + let images = + find_paths + (fun p -> + Filename.extension p = ".png" + || Filename.extension p = ".jpeg") + src_manifest + in + let images = + List.map Uri.pct_encode images + |> List.map (fun v -> + "." / id + / (Uri.with_query (Uri.of_string "serve") + [ ("file", [ v ]) ] + |> Uri.to_string)) + in + let table = + let csvs = + find_paths (fun p -> Filename.extension p = ".csv") src_manifest + in + List.fold_left + (fun acc data -> + ( In_channel.with_open_bin (Filename.concat src_dir data) + @@ fun ic -> Csv.load_in ic ) + :: acc) + [] csvs + in + ( Obuilder.Manifest.sexp_of_t src_manifest + |> Sexplib.Sexp.to_string_hum, + geojsons, + jsons, + images, + table ) + in + let page = + Pages.build ~geojsons ~jsons ~images ~tabular ~manifest ~title:"Build" + ~id ~inputs:[] () + in + let body = + Cohttp_eio.Body.of_string (Htmlit.El.to_string ~doctype:true page) + in + let headers = + (* Otherwise, an nginx reverse proxy will wait for the whole log before sending anything. *) + Cohttp.Header.init_with "X-Accel-Buffering" "no" + in + Cohttp_eio.Server.respond ~status:`OK ~headers ~body () diff --git a/src/lib/server/dune b/src/lib/server/dune index eaa03ba5..2ff39897 100644 --- a/src/lib/server/dune +++ b/src/lib/server/dune @@ -14,4 +14,4 @@ (library (name shark_server) (public_name shark.server) - (libraries shark cohttp-eio htmlit routes)) + (libraries shark cohttp-eio htmlit routes csv)) diff --git a/src/lib/server/pages.ml b/src/lib/server/pages.ml new file mode 100644 index 00000000..f359b758 --- /dev/null +++ b/src/lib/server/pages.ml @@ -0,0 +1,342 @@ +let template title body = + let open Htmlit in + let more_head = + El.splice + [ + El.link + ~at: + [ + At.rel "stylesheet"; + At.href + "https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/base-min.css"; + ] + (); + El.link + ~at: + [ + At.rel "stylesheet"; + At.href + "https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/grids-min.css"; + ] + (); + El.link + ~at: + [ + At.rel "stylesheet"; + At.href + "https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/grids-responsive-min.css"; + ] + (); + El.link + ~at: + [ + At.rel "stylesheet"; + At.href + "https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/buttons-min.css"; + ] + (); + El.link + ~at: + [ + At.rel "stylesheet"; + At.href + "https://cdn.jsdelivr.net/npm/purecss@3.0.0/build/tables-min.css"; + ] + (); + El.link + ~at: + [ + At.rel "stylesheet"; + At.href "https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"; + ] + (); + El.script + ~at:[ At.src "https://unpkg.com/leaflet@1.9.4/dist/leaflet.js" ] + []; + El.style + [ + El.unsafe_raw + {| + body { + margin: 24px + } + + .l-box { + padding: 0em 2em; + } + + .pure-g .pure-u-3-5 div { + border-right: thin solid grey; + } + + #map { width: 100%; height: 800px } + |}; + ]; + ] + in + El.page ~lang:"en" ~more_head ~title body + +let divc class' children = + Htmlit.El.div ~at:[ Htmlit.At.class' class' ] children + +let pure_button ?(disabled = false) href txt = + Htmlit.El.a + ~at: + [ + Htmlit.At.href href; + (if disabled then Htmlit.At.disabled else Htmlit.At.void); + Htmlit.At.class' "pure-button"; + ] + [ Htmlit.El.txt txt ] + +let map geojsons = + let open Htmlit in + let add = + {|fetch(url).then(res => res.json()).then(data => { + // add GeoJSON layer to the map once the file is loaded + var k = L.geoJson(data, { + style: lineStyle, + minZoom: 3, + pointToLayer: function (feature, latlng) { + return L.circleMarker(latlng, geojsonMarkerOptions); + }}); + if (i == 0) { map.setView(k.getBounds().getCenter(), 6) }; + var obj = {}; + // Extracting filepath from query params of URL + var file = new URLSearchParams(url.split("?")[1]).get("file").split("%2F")[1] + obj[file] = k; + return obj; + })|} + in + let jsarr = + Fmt.str "var urls = [ %a ]" + Fmt.(list ~sep:(Fmt.any ", ") (quote string)) + geojsons + in + El.script + [ + El.unsafe_raw + (Fmt.str + {| + var geojsonMarkerOptions = { + radius: 3, + fillColor: "#ff0000", + color: "#000", + weight: 1, + opacity: 0.4, + fillOpacity: 0.4 + }; + var lineStyle = { + "color": "#ff7800", + "weight": 2, + "opacity": 0.3 + }; + var osm = L.tileLayer('https://tile.openstreetmap.org/{z}/{x}/{y}.png', { + minZoom: 3, + maxZoom: 19, + attribution: '© OpenStreetMap contributors' + }) + var map = L.map('map', { + center: [-5.006552150273841, -1.1807189565615772e-05], + preferCanvas: true, + zoom: 13, + minZoom: 3, + maxZoom: 13, + layers: [ osm ] + }) + %s + var proms = urls.map((url, i) => %s) + Promise.all(proms).then((layers) => { + var layers = layers.reduce(((r, c) => Object.assign(r, c)), {}); + console.log(layers); + L.control.layers({ "OSM": osm }, layers, {collapsed: false}).addTo(map) + }) + |} + jsarr add); + ] + +module Table = struct + type t = string list list + + let render t = + let open Htmlit in + let row ?(header = false) s = + let el = if header then El.th else El.td in + El.tr (List.map (fun s -> el [ El.txt s ]) s) + in + match t with + | header :: data -> + let header = El.thead [ row ~header:true header ] in + El.table + ~at:[ At.class' "pure-table" ] + [ header; El.tbody (List.map (fun r -> row r) data) ] + | _ -> El.txt "Something went wrong rendering the tabluar data" +end + +let build ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) ~title + ~id ~inputs ~manifest () = + let open Htmlit in + El.splice + [ + divc "pure-g" + [ + divc "pure-u-3-5" + [ + divc "l-box" + [ + El.h2 [ El.txt "Data and Build Information" ]; + El.em [ El.txt ("Job ID: " ^ id) ]; + El.p + [ + pure_button "#" "Download Data"; + pure_button ~disabled:true "#" "Run Shell"; + pure_button ~disabled:true "#" "Run Notebook"; + ]; + El.p + [ + El.txt + "The following files were generated during this build \ + step. Clicking 'Download Data' will zip these up and \ + begin a download for them. This is only really \ + feasible on relatively small datasets."; + ]; + El.pre + ~at:[ At.style "height:300px; overflow-y:scroll" ] + [ El.code [ El.txt manifest ] ]; + (match geojsons with + | [] -> El.void + | _ -> + El.splice + [ + El.h2 [ El.txt "Maps" ]; + El.p + [ + El.txt + "The map below plots the obviously plottable \ + pieces of data that we could find. For now \ + this is any GeoJSON data files."; + ]; + El.p + [ + El.em + [ + El.txt + "Note we've zoomed into bounding box of \ + the first piece of data, the map may \ + contain others!"; + ]; + ]; + El.div ~at:[ At.id "map" ] []; + ]); + (match images with + | [] -> El.void + | urls -> + El.splice + ([ + El.h2 [ El.txt "Images" ]; + El.p + [ + El.txt + "Images generated as output data, we only \ + render PNG or JPEG images for now as they \ + are likely to be small enough."; + ]; + ] + @ List.map + (fun url -> + El.img ~at:[ At.v "width" "100%"; At.src url ] ()) + urls)); + (match jsons with + | [] -> El.void + | contents -> + El.splice + ([ + El.h2 [ El.txt "JSON Files" ]; + El.p + [ + El.txt + "Raw JSON files that are not geospatial in \ + nature."; + ]; + ] + @ List.map + (fun content -> + El.pre [ El.code [ El.txt content ] ]) + contents)); + (match tabular with + | [] -> El.void + | datas -> + El.splice + [ + El.h2 [ El.txt "Tabular Data" ]; + El.p + [ + El.txt + "Tables for any CSV files found in the output \ + data directory."; + ]; + El.splice (List.map Table.render datas); + ]); + ]; + ]; + divc "pure-u-2-5" + [ + divc "l-box" + [ + El.h3 [ El.txt "Build Summary" ]; + El.p + [ + El.txt + "The following command was the last to be run in this \ + pipeline step."; + ]; + El.pre + [ + El.code [ El.txt "python -m methods.matching.find_pairs" ]; + ]; + El.h3 [ El.txt "Data Dependencies" ]; + El.p + [ + El.txt + {| + The following list is the immediate data dependencies of this build step. + Put another way, these build outputs were made available to this build step + in read-only mode. The actual code may or may not have used the data. + |}; + ]; + El.ul + (List.map + (fun i -> + El.li + [ El.a ~at:[ At.href "#" ] [ El.code [ El.txt i ] ] ]) + inputs); + El.h3 [ El.txt "Build Specifications" ]; + El.p + [ + El.txt + "The build specification is a bit like a Dockerfile. \ + In this form it is very raw and specific to how the \ + dataflow pipeline works, but it has been copied here \ + for your convenience."; + ]; + El.pre + [ + El.code + [ + El.txt + {|((from alpine) (run (shell "echo 'hello world'")))|}; + ]; + ]; + El.h3 [ El.txt "Logs" ]; + El.p + [ + El.txt + {|Raw logs from the build of this particular pipeline step. The data here is very raw, but can help explain how the data was produced.|}; + ]; + pure_button "./logs" "Raw Logs"; + ]; + ]; + ]; + (match geojsons with [] -> El.void | lst -> map lst); + ] + |> template title diff --git a/src/lib/server/shark_server.ml b/src/lib/server/shark_server.ml index 15476105..64650683 100644 --- a/src/lib/server/shark_server.ml +++ b/src/lib/server/shark_server.ml @@ -446,6 +446,8 @@ let serve_dot proc _req body = let png = run_dot proc txt |> Base64.encode_string in respond_txt png +let serve_data store id = Build.render store id + let edit_routes ~proc md_file (_conn : Cohttp_eio.Server.conn) request body = let open Routes in [ @@ -461,6 +463,7 @@ let router ~proc ~fs ~store md_file (conn : Cohttp_eio.Server.conn) request body [ route nil (serve md_file); route (s "logs" / str /? nil) (serve_logs fs store); + route (s "data" / str /? nil) (serve_data store); route (s "files" / str /? nil) (fun hash -> serve_files fs store hash None); diff --git a/vendor/obuilder b/vendor/obuilder index 5f73e236..3242b3da 160000 --- a/vendor/obuilder +++ b/vendor/obuilder @@ -1 +1 @@ -Subproject commit 5f73e236da76af2a1e054d77edef2bbde82c1a74 +Subproject commit 3242b3da525a0902746ddd3918bd4bbbfa0e9702 From 7dacb5ce93052ce8fbef30ec62e6bc4d42915844 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sat, 6 Jul 2024 18:05:11 +0100 Subject: [PATCH 2/6] Add info page with data download --- src/lib/server/build.ml | 15 +++- src/lib/server/pages.ml | 152 +++++++++++++++++++++++---------- src/lib/server/shark_server.ml | 37 +++++++- 3 files changed, 154 insertions(+), 50 deletions(-) diff --git a/src/lib/server/build.ml b/src/lib/server/build.ml index df7640f9..315c484c 100644 --- a/src/lib/server/build.ml +++ b/src/lib/server/build.ml @@ -10,7 +10,18 @@ let find_paths pred paths = (* Rendering Builds *) let render (Obuilder.Store_spec.Store ((module Store), store)) id = - let result = Lwt_eio.run_lwt (fun () -> Store.result store id) in + let result, inputs = + Lwt_eio.run_lwt (fun () -> + let open Lwt.Syntax in + let* result = Store.result store id in + let+ inputs = Store.get_meta store id ":obuilder-run-input" in + ( result, + match inputs with + | Some inputs -> + Some + (Obuilder.S.run_input_of_sexp (Sexplib.Sexp.of_string inputs)) + | None -> None )) + in match result with | None -> Cohttp_eio.Server.respond_string ~status:`OK ~body:("No result for " ^ id) @@ -79,7 +90,7 @@ let render (Obuilder.Store_spec.Store ((module Store), store)) id = in let page = Pages.build ~geojsons ~jsons ~images ~tabular ~manifest ~title:"Build" - ~id ~inputs:[] () + ~id ?inputs () in let body = Cohttp_eio.Body.of_string (Htmlit.El.to_string ~doctype:true page) diff --git a/src/lib/server/pages.ml b/src/lib/server/pages.ml index f359b758..dd62bc7d 100644 --- a/src/lib/server/pages.ml +++ b/src/lib/server/pages.ml @@ -68,6 +68,67 @@ let template title body = .pure-g .pure-u-3-5 div { border-right: thin solid grey; } + :root + { font-size: 100%; + /* font-synthesis: none; */ + -webkit-text-size-adjust: none; + + --font_headings: system-ui, sans-serif; + --font_body: system-ui, sans-serif; + --font_mono: monospace; + + --font_m: 1rem; --leading_m: 1.5rem; + --font_s: 0.82rem; + --font_l: 1.125rem; --leadig_l: 1.34rem; + --font_xl: 1.5rem; --leading_xl: 1.8rem; + --font_xxl: 2.5rem; --leading_xxl: 3rem; + + --font_mono_ratio: + /* mono / body size, difficult to find a good cross-browser value */ + 0.92; + --leading_mono_m: calc(var(--leading_m) * var(--font_mono_ratio)); + + --sp_xxs: calc(0.25 * var(--leading_m)); + --sp_xs: calc(0.5 * var(--leading_m)); + --sp_s: calc(0.75 * var(--leading_m)); + --sp_m: var(--leading_m); + --sp_l: calc(1.125 * var(--leading_m)); + --sp_xl: calc(1.5 * var(--leading_m)); + --sp_xxl: calc(2.0 * var(--leading_m)); + + --measure_m: 73ch; + --page_inline_pad: var(--sp_m); + --page_block_pad: var(--sp_xl); + + --blockquote_border: 2px solid #ACACAC; + --rule_border: 1px solid #CACBCE; + --heading_border: 1px solid #EAECEF; + --table_cell_pad: 0.4em; + --table_hover: #f5f5f5; + --table_sep: #efefef; + --table_cell_inline_pad: 0.625em; + --table_cell_block_pad: 0.25em; + + --code_span_bg: #EFF1F3; + --code_span_inline_pad: 0.35ch; + --code_block_bg: #F6F8FA; + --code_block_bleed: 0.8ch; + --code_block_block_pad: 1ch; + + --a_fg: #0969DA; + --a_fg_hover: #1882ff; + --a_visited: #8E34A5; + --target_color: #FFFF96; + } + + pre + { line-height: var(--leading_mono_m); + white-space: pre-wrap; + overflow-wrap: break-word; + background-color: var(--code_block_bg); + padding-block: var(--code_block_block_pad); + padding-inline: var(--code_block_bleed); + margin-inline: calc(-1.0 * var(--code_block_bleed)) } #map { width: 100%; height: 800px } |}; @@ -173,8 +234,8 @@ module Table = struct | _ -> El.txt "Something went wrong rendering the tabluar data" end -let build ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) ~title - ~id ~inputs ~manifest () = +let build ?inputs ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) + ~title ~id ~manifest () = let open Htmlit in El.splice [ @@ -188,7 +249,7 @@ let build ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) ~title El.em [ El.txt ("Job ID: " ^ id) ]; El.p [ - pure_button "#" "Download Data"; + pure_button (Fmt.str "/download/%s" id) "Download Data"; pure_button ~disabled:true "#" "Run Shell"; pure_button ~disabled:true "#" "Run Notebook"; ]; @@ -284,56 +345,55 @@ let build ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) ~title divc "l-box" [ El.h3 [ El.txt "Build Summary" ]; - El.p - [ - El.txt - "The following command was the last to be run in this \ - pipeline step."; - ]; - El.pre - [ - El.code [ El.txt "python -m methods.matching.find_pairs" ]; - ]; - El.h3 [ El.txt "Data Dependencies" ]; - El.p - [ - El.txt - {| - The following list is the immediate data dependencies of this build step. - Put another way, these build outputs were made available to this build step - in read-only mode. The actual code may or may not have used the data. - |}; - ]; - El.ul - (List.map - (fun i -> - El.li - [ El.a ~at:[ At.href "#" ] [ El.code [ El.txt i ] ] ]) - inputs); - El.h3 [ El.txt "Build Specifications" ]; - El.p - [ - El.txt - "The build specification is a bit like a Dockerfile. \ - In this form it is very raw and specific to how the \ - dataflow pipeline works, but it has been copied here \ - for your convenience."; - ]; - El.pre - [ - El.code + El.div + (match inputs with + | None -> [] + | Some (i : Obuilder.S.run_input) -> + let base = + El.div + [ + El.p [ El.txt "Base Image" ]; + El.pre [ El.code [ El.txt i.base ] ]; + ] + in + let cmd = + El.div + [ + El.p [ El.txt "The command run was:" ]; + El.pre [ El.code [ El.txt i.cmd ] ]; + ] + in + let roms = + if i.rom = [] then + El.p [ El.txt "No data dependencies" ] + else + El.ul + (List.map + (fun (r : Obuilder_spec.Rom.t) -> + match r.kind with + | `Build (hash, _dir) -> + El.li + [ + El.a + ~at: + [ + At.href + (Fmt.str "/data/%s" hash); + ] + [ El.txt (String.sub hash 0 12) ]; + ]) + i.rom) + in [ - El.txt - {|((from alpine) (run (shell "echo 'hello world'")))|}; - ]; - ]; + base; cmd; El.h3 [ El.txt "Data Dependencies" ]; roms; + ]); El.h3 [ El.txt "Logs" ]; El.p [ El.txt {|Raw logs from the build of this particular pipeline step. The data here is very raw, but can help explain how the data was produced.|}; ]; - pure_button "./logs" "Raw Logs"; + pure_button (Fmt.str "/logs/%s" id) "Raw Logs"; ]; ]; ]; diff --git a/src/lib/server/shark_server.ml b/src/lib/server/shark_server.ml index 64650683..7bfe63d6 100644 --- a/src/lib/server/shark_server.ml +++ b/src/lib/server/shark_server.ml @@ -237,7 +237,7 @@ let custom_document_renderer _ = function ~at: [ At.v "target" "_blank"; - At.href (Fmt.str "/logs/%s" hash); + At.href (Fmt.str "/data/%s" hash); At.style "text-decoration:none;"; ] [ @@ -248,7 +248,7 @@ let custom_document_renderer _ = function "display:inline-flex;align-items:center;padding:0.2em \ 0.4em"; ] - [ log; El.nbsp; El.txt (Fmt.str "Logs") ]; + [ log; El.nbsp; El.txt (Fmt.str "Info") ]; ]; El.a ~at: @@ -448,6 +448,38 @@ let serve_dot proc _req body = let serve_data store id = Build.render store id +let with_file f fn = + let open Lwt.Infix in + Lwt_unix.openfile f [ Unix.O_RDWR; Unix.O_CREAT ] 0o644 >>= fun fd -> + Lwt.finalize (fun () -> fn (f, fd)) (fun () -> Lwt_unix.close fd) + +let download ~fs (Obuilder.Store_spec.Store ((module Store), store)) id = + match Lwt_eio.run_lwt @@ fun () -> Store.result store id with + | None -> Cohttp_eio.Server.respond_string ~status:`Not_found ~body:"" () + | Some src_dir -> ( + Logs.info (fun f -> f "Download for %s" src_dir); + let src_dir = Filename.concat src_dir "rootfs" in + match Obuilder.Manifest.generate ~exclude:[] ~src_dir "data" with + | Error (`Msg m) -> + Cohttp_eio.Server.respond_string ~status:`Bad_request + ~body:("Failed data zip: " ^ m) () + | Ok src_manifest -> + let fname = Filename.temp_file "tmf-" ".zip" in + let tar () = + with_file fname @@ fun (_, fd) -> + Obuilder.Tar_transfer.send_files ~src_dir + ~src_manifest:[ src_manifest ] ~dst_dir:"" ~to_untar:fd + ~user:(`Unix Obuilder_spec.{ uid = 1000; gid = 1000 }) + in + let headers = + Http.Header.add_opt_unless_exists None "content-type" + "application/zip" + in + (* Some respond_file API would be nice here *) + let () = Lwt_eio.run_lwt tar in + let body = Cohttp_eio.Body.of_string Eio.Path.(load (fs / fname)) in + Cohttp_eio.Server.respond ~status:`OK ~headers ~body ()) + let edit_routes ~proc md_file (_conn : Cohttp_eio.Server.conn) request body = let open Routes in [ @@ -464,6 +496,7 @@ let router ~proc ~fs ~store md_file (conn : Cohttp_eio.Server.conn) request body route nil (serve md_file); route (s "logs" / str /? nil) (serve_logs fs store); route (s "data" / str /? nil) (serve_data store); + route (s "download" / str /? nil) (download ~fs store); route (s "files" / str /? nil) (fun hash -> serve_files fs store hash None); From 495431a6a15c3149b324244e68349567e2326ff9 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sat, 6 Jul 2024 18:40:55 +0100 Subject: [PATCH 3/6] Add geojson example --- specs/shark.md | 1 + specs/shark.out.md | 13 +++++++------ src/lib/server/build.ml | 7 +------ src/lib/server/dune | 2 +- src/lib/server/pages.ml | 4 ++-- src/lib/server/shark_server.ml | 26 ++++++++++++++++++++------ 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/specs/shark.md b/specs/shark.md index cf13ce31..e32de942 100644 --- a/specs/shark.md +++ b/specs/shark.md @@ -20,6 +20,7 @@ using that environment. ```shark-run:gdal-env $ gdalinfo --version > /data/gdal.version +$ curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson ``` Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire diff --git a/specs/shark.out.md b/specs/shark.out.md index b9ca21fd..5bcff1c4 100644 --- a/specs/shark.out.md +++ b/specs/shark.out.md @@ -8,7 +8,7 @@ built and can be referenced as the context for future `shark-run` blocks. ## Shark Build -```shark-build:gdal-env:ec610a45b8d858c2eba37fd40dd1764890828557c1c43fa84ec88c7fcdc087c1 +```shark-build:gdal-env ((from osgeo/gdal:ubuntu-small-3.6.3) (run (shell "mkdir -p /data && echo 'Something for the log!'"))) ``` @@ -18,15 +18,16 @@ using that environment. ## Shark Run -```shark-run:gdal-env:1dd3d7fdb8f1f485dd5aa0d5f383209a60aca98e67552d03a54c99be8b610eca -$ gdalinfo --version > /data/gdal.version +```shark-run:gdal-env:bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b +gdalinfo --version > /data/gdal.version +curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson ``` Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire up `/data/gdal.version` into the container. -```shark-run:gdal-env:e02469d800253ccf95e53b583e4a91465375a4e41479a67408331ecdeedb713e -$ cat /data/gdal.version +```shark-run:gdal-env:a4438aaea4711bfb05d666f91bb6fa8ba69d0bcd3e4398027538b3346855273b +cat /shark/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/gdal.version GDAL 3.6.3, released 2023/03/07 ``` @@ -38,5 +39,5 @@ this will publish to a `_shark` directory in the current working directory. Use conventions to export data blobs. ```shark-publish -/data/gdal.version +/obuilder-zfs/result/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/.zfs/snapshot/snap/rootfs/data/gdal.version ``` diff --git a/src/lib/server/build.ml b/src/lib/server/build.ml index 315c484c..c461c250 100644 --- a/src/lib/server/build.ml +++ b/src/lib/server/build.ml @@ -41,12 +41,7 @@ let render (Obuilder.Store_spec.Store ((module Store), store)) id = find_paths (fun p -> Filename.extension p = ".json") src_manifest in let geojsons = - List.map Uri.pct_encode geojsons - |> List.map (fun v -> - "." / id - / (Uri.with_query (Uri.of_string "serve") - [ ("file", [ v ]) ] - |> Uri.to_string)) + List.map (fun g -> Fmt.str "/file/%s/%s" id g) geojsons in let jsons = List.map diff --git a/src/lib/server/dune b/src/lib/server/dune index 2ff39897..24b03afb 100644 --- a/src/lib/server/dune +++ b/src/lib/server/dune @@ -14,4 +14,4 @@ (library (name shark_server) (public_name shark.server) - (libraries shark cohttp-eio htmlit routes csv)) + (libraries shark cohttp-eio htmlit routes csv magic-mime)) diff --git a/src/lib/server/pages.ml b/src/lib/server/pages.ml index dd62bc7d..c2a92bdc 100644 --- a/src/lib/server/pages.ml +++ b/src/lib/server/pages.ml @@ -164,7 +164,7 @@ let map geojsons = if (i == 0) { map.setView(k.getBounds().getCenter(), 6) }; var obj = {}; // Extracting filepath from query params of URL - var file = new URLSearchParams(url.split("?")[1]).get("file").split("%2F")[1] + var file = url.split("/")[url.split("/").length - 1] obj[file] = k; return obj; })|} @@ -262,7 +262,7 @@ let build ?inputs ?(geojsons = []) ?(jsons = []) ?(images = []) ?(tabular = []) feasible on relatively small datasets."; ]; El.pre - ~at:[ At.style "height:300px; overflow-y:scroll" ] + ~at:[ At.style "max-height:300px; overflow-y:scroll" ] [ El.code [ El.txt manifest ] ]; (match geojsons with | [] -> El.void diff --git a/src/lib/server/shark_server.ml b/src/lib/server/shark_server.ml index 7bfe63d6..aeb83e21 100644 --- a/src/lib/server/shark_server.ml +++ b/src/lib/server/shark_server.ml @@ -453,6 +453,12 @@ let with_file f fn = Lwt_unix.openfile f [ Unix.O_RDWR; Unix.O_CREAT ] 0o644 >>= fun fd -> Lwt.finalize (fun () -> fn (f, fd)) (fun () -> Lwt_unix.close fd) +let respond_file ~fs path = + let mime = Magic_mime.lookup path in + let headers = Http.Header.add_opt_unless_exists None "content-type" mime in + let body = Cohttp_eio.Body.of_string Eio.Path.(load (fs / path)) in + Cohttp_eio.Server.respond ~status:`OK ~headers ~body () + let download ~fs (Obuilder.Store_spec.Store ((module Store), store)) id = match Lwt_eio.run_lwt @@ fun () -> Store.result store id with | None -> Cohttp_eio.Server.respond_string ~status:`Not_found ~body:"" () @@ -471,14 +477,17 @@ let download ~fs (Obuilder.Store_spec.Store ((module Store), store)) id = ~src_manifest:[ src_manifest ] ~dst_dir:"" ~to_untar:fd ~user:(`Unix Obuilder_spec.{ uid = 1000; gid = 1000 }) in - let headers = - Http.Header.add_opt_unless_exists None "content-type" - "application/zip" - in (* Some respond_file API would be nice here *) let () = Lwt_eio.run_lwt tar in - let body = Cohttp_eio.Body.of_string Eio.Path.(load (fs / fname)) in - Cohttp_eio.Server.respond ~status:`OK ~headers ~body ()) + respond_file ~fs fname) + +let serve_file ~fs (Obuilder.Store_spec.Store ((module Store), store)) id path = + match Lwt_eio.run_lwt @@ fun () -> Store.result store id with + | None -> Cohttp_eio.Server.respond_string ~status:`Not_found ~body:"" () + | Some src_dir -> + let src_dir = Filename.concat src_dir "rootfs" in + let path = Filename.concat src_dir path in + respond_file ~fs path let edit_routes ~proc md_file (_conn : Cohttp_eio.Server.conn) request body = let open Routes in @@ -496,6 +505,11 @@ let router ~proc ~fs ~store md_file (conn : Cohttp_eio.Server.conn) request body route nil (serve md_file); route (s "logs" / str /? nil) (serve_logs fs store); route (s "data" / str /? nil) (serve_data store); + route + (s "file" / str /? wildcard) + (fun id path -> + let dir = Parts.wildcard_match path in + serve_file ~fs store id dir); route (s "download" / str /? nil) (download ~fs store); route (s "files" / str /? nil) From 6205e7d12b60eb8c5f8cc3e451308460e4cf8407 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sat, 6 Jul 2024 19:55:41 +0100 Subject: [PATCH 4/6] Expose run_input and fix opam files --- shark.opam | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shark.opam b/shark.opam index 36067555..d354f1ca 100644 --- a/shark.opam +++ b/shark.opam @@ -24,6 +24,8 @@ depends: [ "htmlit" "routes" "code-mirror" + "magic-mime" + "csv" # Container-image deps / OBuilder "mirage-crypto-rng" From 55bcfbf1e8f23a2dee5d5851c45912128208c316 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sun, 7 Jul 2024 10:23:28 +0100 Subject: [PATCH 5/6] Fix build hash bug and make spec more repro --- specs/shark.md | 2 +- specs/shark.out.md | 12 ++++++------ src/lib/md.ml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/shark.md b/specs/shark.md index e32de942..cd8a7331 100644 --- a/specs/shark.md +++ b/specs/shark.md @@ -9,7 +9,7 @@ built and can be referenced as the context for future `shark-run` blocks. ## Shark Build ```shark-build:gdal-env -((from osgeo/gdal:ubuntu-small-3.6.3) +((from ghcr.io/osgeo/gdal:ubuntu-small-3.6.3@sha256:bfa7915a3ef942b4f6f61223ee57eadbb469d6fb4a5fbf562286d1473f15eaab) (run (shell "mkdir -p /data && echo 'Something for the log!'"))) ``` diff --git a/specs/shark.out.md b/specs/shark.out.md index 5bcff1c4..2173a66f 100644 --- a/specs/shark.out.md +++ b/specs/shark.out.md @@ -8,8 +8,8 @@ built and can be referenced as the context for future `shark-run` blocks. ## Shark Build -```shark-build:gdal-env -((from osgeo/gdal:ubuntu-small-3.6.3) +```shark-build:gdal-env:4213afafe74bc720d4bb210f21c97d54a361a80a838c248f26dd7dc019a40ac2 +((from ghcr.io/osgeo/gdal:ubuntu-small-3.6.3@sha256:bfa7915a3ef942b4f6f61223ee57eadbb469d6fb4a5fbf562286d1473f15eaab) (run (shell "mkdir -p /data && echo 'Something for the log!'"))) ``` @@ -18,7 +18,7 @@ using that environment. ## Shark Run -```shark-run:gdal-env:bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b +```shark-run:gdal-env:8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509 gdalinfo --version > /data/gdal.version curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson ``` @@ -26,8 +26,8 @@ curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-oc Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire up `/data/gdal.version` into the container. -```shark-run:gdal-env:a4438aaea4711bfb05d666f91bb6fa8ba69d0bcd3e4398027538b3346855273b -cat /shark/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/gdal.version +```shark-run:gdal-env:101cf72cd986ca89f23d87c6af4c86908385fd977ebdd4f010ebf6ae8d0b04c6 +cat /shark/8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509/gdal.version GDAL 3.6.3, released 2023/03/07 ``` @@ -39,5 +39,5 @@ this will publish to a `_shark` directory in the current working directory. Use conventions to export data blobs. ```shark-publish -/obuilder-zfs/result/bf3b13fbfff681b941770ca0e89048afd3e185a2a5f793b63d8728347798f60b/.zfs/snapshot/snap/rootfs/data/gdal.version +/obuilder-zfs/result/8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509/.zfs/snapshot/snap/rootfs/data/gdal.version ``` diff --git a/src/lib/md.ml b/src/lib/md.ml index 79b6de2c..de5c3ec3 100644 --- a/src/lib/md.ml +++ b/src/lib/md.ml @@ -71,7 +71,7 @@ let process_build_block ?(src_dir = ".") ?hb in Ast.Hyperblock.update_hash hb id; let new_code_block = - let info_string = Block.to_info_string block in + let info_string = Block.to_info_string block_with_hash in Cmarkit.Block.Code_block.make ~info_string:(info_string, Cmarkit.Meta.none) (Cmarkit.Block.Code_block.code code_block) From 5078a56e34a1ecb219baa958d483f48087a6726d Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sun, 7 Jul 2024 10:27:44 +0100 Subject: [PATCH 6/6] Promote tests --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3eb3540a..58ff4f16 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ the [promoted output version](./specs/shark.out.md). $ patdiff -ascii specs/shark.md specs/shark.out.md ------ specs/shark.md ++++++ specs/shark.out.md -@|-1,40 +1,42 ============================================================ +@|-1,41 +1,43 ============================================================ | |# Markdown Shark Support | @@ -72,8 +72,8 @@ $ patdiff -ascii specs/shark.md specs/shark.out.md |## Shark Build | -|```shark-build:gdal-env -+|```shark-build:gdal-env:ec610a45b8d858c2eba37fd40dd1764890828557c1c43fa84ec88c7fcdc087c1 - |((from osgeo/gdal:ubuntu-small-3.6.3) ++|```shark-build:gdal-env:4213afafe74bc720d4bb210f21c97d54a361a80a838c248f26dd7dc019a40ac2 + |((from ghcr.io/osgeo/gdal:ubuntu-small-3.6.3@sha256:bfa7915a3ef942b4f6f61223ee57eadbb469d6fb4a5fbf562286d1473f15eaab) | (run (shell "mkdir -p /data && echo 'Something for the log!'"))) |``` | @@ -83,16 +83,20 @@ $ patdiff -ascii specs/shark.md specs/shark.out.md |## Shark Run | -|```shark-run:gdal-env -+|```shark-run:gdal-env:1dd3d7fdb8f1f485dd5aa0d5f383209a60aca98e67552d03a54c99be8b610eca - |$ gdalinfo --version > /data/gdal.version +-|$ gdalinfo --version > /data/gdal.version +-|$ curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson ++|```shark-run:gdal-env:8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509 ++|gdalinfo --version > /data/gdal.version ++|curl -s https://france-geojson.gregoiredavid.fr/repo/regions/occitanie/region-occitanie.geojson > /data/region-occitanie.geojson |``` | |Shark keeps track of inputs and outputs. In the next code block, Shark knows to wire |up `/data/gdal.version` into the container. | -|```shark-run:gdal-env -+|```shark-run:gdal-env:e02469d800253ccf95e53b583e4a91465375a4e41479a67408331ecdeedb713e - |$ cat /data/gdal.version +-|$ cat /data/gdal.version ++|```shark-run:gdal-env:101cf72cd986ca89f23d87c6af4c86908385fd977ebdd4f010ebf6ae8d0b04c6 ++|cat /shark/8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509/gdal.version +|GDAL 3.6.3, released 2023/03/07 +| |``` @@ -104,7 +108,8 @@ $ patdiff -ascii specs/shark.md specs/shark.out.md |conventions to export data blobs. | |```shark-publish - |/data/gdal.version +-|/data/gdal.version ++|/obuilder-zfs/result/8113359387d02c4e29df7013f0cd2d699c4f1303d99fe065204b579baf0dd509/.zfs/snapshot/snap/rootfs/data/gdal.version |``` [1] ```