From 2feb22365d17e7ff3cc1a5677b7df199648e3d22 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Wed, 13 Mar 2024 16:05:40 -0500 Subject: [PATCH] move Makie support to an extension (#109) * move Makie support to an extension * test fix for Julia 1.10 * update CI --- .github/workflows/CI.yml | 31 +-- Project.toml | 11 +- docs/Project.toml | 1 + docs/src/plotting.md | 8 + ext/LighthouseMakieExt.jl | 480 ++++++++++++++++++++++++++++++++++++++ src/Lighthouse.jl | 6 +- src/plotting.jl | 475 +++---------------------------------- src/row.jl | 1 - test/row.jl | 7 +- 9 files changed, 562 insertions(+), 458 deletions(-) create mode 100644 ext/LighthouseMakieExt.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index acfc1e4..2b40436 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -7,6 +7,11 @@ on: tags: - v* pull_request: +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }}-TrialReportCI + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - Makie - ${{ matrix.makie }} @@ -16,7 +21,7 @@ jobs: matrix: version: - '1' - - '1.7' + - '1.9' - '1.6' os: - ubuntu-latest @@ -24,21 +29,16 @@ jobs: - x64 makie: - '0.20' - env: - JULIA_NUM_THREADS: 2 steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v2 + - uses: julia-actions/cache@v1 with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-artifacts-${{ hashFiles('**/Project.toml') }} - restore-keys: ${{ runner.os }}-test-artifacts + cache-compiled: true + cache-name: ${{ github.workflow }}-${{ github.job }}-${{ matrix.os }}-${{ matrix.version }}-${{ matrix.arch }}-${{ matrix.makie }} - name: "Install Makie and instantiate project" shell: julia --color=yes --project {0} run: | @@ -54,17 +54,22 @@ jobs: env: PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v4 with: file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} docs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: - version: '1.6' + version: '1.10' + - uses: julia-actions/cache@v1 + with: + cache-compiled: true + cache-name: ${{ github.workflow }}-${{ github.job }} - run: | julia --project=docs -e ' using Pkg diff --git a/Project.toml b/Project.toml index 87b0ec4..3f80de6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Lighthouse" uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59" authors = ["Beacon Biosignals, Inc."] -version = "0.16.0" +version = "0.17.0" [deps] ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" @@ -18,6 +18,12 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" TensorBoardLogger = "899adc3e-224a-11e9-021f-63837185c80f" +[weakdeps] +Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" + +[extensions] +LighthouseMakieExt = ["Makie"] + [compat] Arrow = "2.3" ArrowTypes = "1, 2" @@ -33,8 +39,9 @@ julia = "1.6" [extras] Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45" CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "Arrow", "CairoMakie", "StableRNGs"] +test = ["Test", "Arrow", "CairoMakie", "Makie", "StableRNGs"] diff --git a/docs/Project.toml b/docs/Project.toml index 54f344c..bb49311 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,6 +1,7 @@ [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Lighthouse = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" [compat] diff --git a/docs/src/plotting.md b/docs/src/plotting.md index ce9d57b..5606b68 100644 --- a/docs/src/plotting.md +++ b/docs/src/plotting.md @@ -1,3 +1,10 @@ +!!! note + All plotting functions require a valid Makie backend, e.g. CairoMakie, to be loaded. + If there is no backend loaded, then functions won't do anything interesting. + On Julia 1.9+, Makie is a weak dependency and so won't incur any compilation/dependency + cost without a backend. On Julia < 1.9, Makie will still be loaded, but without a + backend, the resulting figure will not be rendered. + # Confusion matrices ```@docs @@ -82,6 +89,7 @@ Note that all curve plot types accepts these types: Lighthouse.XYVector Lighthouse.SeriesCurves ``` + ## Theming All generic series and axis attributes can be themed via `SeriesPlot.Series` / `SeriesPlot.Axis`. diff --git a/ext/LighthouseMakieExt.jl b/ext/LighthouseMakieExt.jl new file mode 100644 index 0000000..55e4ed6 --- /dev/null +++ b/ext/LighthouseMakieExt.jl @@ -0,0 +1,480 @@ +module LighthouseMakieExt + +using Makie +using Lighthouse +using Printf + +using Lighthouse: has_value, evaluation_metrics + +using Lighthouse: XYVector, SeriesCurves, NumberLike, NumberVector, NumberMatrix + +# all the methods we're actually defining and which call each other +using Lighthouse: evaluation_metrics_plot, plot_confusion_matrix, plot_confusion_matrix!, + plot_reliability_calibration_curves, plot_reliability_calibration_curves!, + plot_binary_discrimination_calibration_curves, + plot_binary_discrimination_calibration_curves!, plot_pr_curves, + plot_pr_curves!, plot_roc_curves, plot_roc_curves!, + plot_kappas, plot_kappas!, evaluation_metrics_plot + +get_parent(x::Makie.GridPosition) = x.layout.parent + +##### +##### Helpers for theming and color generation...May want to move them to Colors.jl / Makie.jl +##### +using Makie.Colors: LCHab, distinguishable_colors, RGB, Colorant + +function get_theme(scene, key1::Symbol, key2::Symbol; defaults...) + return get_theme(Makie.get_scene(scene), key1, key2; defaults...) +end + +function get_theme(fig::GridPosition, key1::Symbol, key2::Symbol; defaults...) + return get_theme(get_parent(fig), key1, key2; defaults...) +end + +# This function helps us to get the theme from a scene, that we can apply to our plotting functions +function get_theme(scene::Scene, key1::Symbol, key2::Symbol; defaults...) + scene_theme = theme(scene) + sub_theme = get(scene_theme, key1, Theme()) + # The priority is key1.key2 > defaults > key2 + # this way defaults are overwritten by theme options specifically set for our theme. + # Consider Kappas.Axis for key1/2, what we want is, that if there are defaults in Kappas.Axis + # they should overwrite our generic lighthouse defaults. But anything not specified in Kappas.Axis/defaults, + # should fall back to scene_theme.Axis + return merge(get(sub_theme, key2, Theme()), Theme(; defaults...), + get(scene_theme, key2, Theme())) +end + +function high_contrast(background_color::Colorant, target_color::Colorant; + # chose from whole lightness spectrum + lchoices=range(0; stop=100, length=15)) + target = LCHab(target_color) + color = distinguishable_colors(1, [RGB(background_color)]; dropseed=true, + lchoices=lchoices, + cchoices=[target.c], hchoices=[target.h]) + return RGBAf(color[1], Makie.Colors.alpha(target_color)) +end + +function series_plot!(subfig::GridPosition, per_class_pr_curves::SeriesCurves, + class_labels::Union{Nothing,AbstractVector{String}}; legend=:lt, + title="No title", + xlabel="x label", ylabel="y label", solid_color=nothing, + color=nothing, + linewidth=nothing, scatter=NamedTuple()) + axis_theme = get_theme(subfig, :SeriesPlot, :Axis; title=title, titlealign=:left, + xlabel=xlabel, + ylabel=ylabel, aspect=AxisAspect(1), + xticks=0:0.2:1, yticks=0.2:0.2:1) + + ax = Axis(subfig; axis_theme...) + # Not the most elegant, but this way we can let the theming to the Series theme, or + # pass it through explicitely + series_theme = get_theme(subfig, :SeriesPlot, :Series; scatter...) + isnothing(solid_color) || (series_theme[:solid_color] = solid_color) + isnothing(color) || (series_theme[:color] = color) + isnothing(linewidth) || (series_theme[:linewidth] = linewidth) + series_theme = merge(series_theme, Attributes(; scatter...)) + hidedecorations!(ax; label=false, ticklabels=false, grid=false) + limits!(ax, 0, 1, 0, 1) + Makie.series!(ax, per_class_pr_curves; labels=class_labels, series_theme...) + if !isnothing(legend) + axislegend(ax; position=legend) + end + return ax +end + +function Lighthouse.plot_pr_curves!(subfig::GridPosition, per_class_pr_curves::SeriesCurves, + class_labels::Union{Nothing,AbstractVector{String}}; + legend=:lt, + title="PR curves", + xlabel="True positive rate", ylabel="Precision", + scatter=NamedTuple(), + solid_color=nothing) + return series_plot!(subfig, per_class_pr_curves, class_labels; legend=legend, + title=title, xlabel=xlabel, + ylabel=ylabel, scatter=scatter, solid_color=solid_color) +end + +function Lighthouse.plot_roc_curves!(subfig::GridPosition, + per_class_roc_curves::SeriesCurves, + per_class_roc_aucs::NumberVector, + class_labels::AbstractVector{<:String}; + legend=:rb, title="ROC curves", + xlabel="False positive rate", + ylabel="True positive rate") + auc_labels = [@sprintf("%s (AUC: %.3f)", class, per_class_roc_aucs[i]) + for (i, class) in enumerate(class_labels)] + + return series_plot!(subfig, per_class_roc_curves, auc_labels; legend=legend, + title=title, xlabel=xlabel, + ylabel=ylabel) +end + +function Lighthouse.plot_reliability_calibration_curves!(subfig::GridPosition, + per_class_reliability_calibration_curves::SeriesCurves, + per_class_reliability_calibration_scores::NumberVector, + class_labels::AbstractVector{String}; + legend=:rb) + calibration_score_labels = map(enumerate(class_labels)) do (i, class) + @sprintf("%s (MSE: %.3f)", class, per_class_reliability_calibration_scores[i]) + end + + scatter_theme = get_theme(subfig, :ReliabilityCalibrationCurves, :Scatter; + marker=Circle, + markersize=5, strokewidth=0) + ideal_theme = get_theme(subfig, :ReliabilityCalibrationCurves, :Ideal; + color=(:black, 0.5), + linestyle=:dash, linewidth=2) + + ax = series_plot!(subfig, per_class_reliability_calibration_curves, + calibration_score_labels; + legend=legend, title="Prediction reliability calibration", + xlabel="Predicted probability bin", ylabel="Fraction of positives", + scatter=scatter_theme) + #TODO: mean predicted value histogram underneath?? Maybe important... + # https://scikit-learn.org/stable/modules/calibration.html + linesegments!(ax, [0, 1], [0, 1]; ideal_theme...) + return ax +end + +function set_from_kw!(theme, key, kw, default) + if haskey(kw, key) + theme[key] = getproperty(kw, key) + elseif !haskey(theme, key) + theme[key] = default + end + return +end + +function Lighthouse.plot_binary_discrimination_calibration_curves!(subfig::GridPosition, + calibration_curve::SeriesCurves, + calibration_score, + per_expert_calibration_curves::Union{SeriesCurves, + Missing}, + per_expert_calibration_scores, + optimal_threshold, + discrimination_class::AbstractString; + kw...) + kw = values(kw) + scatter_theme = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :Scatter; + strokewidth=0) + # Hayaah, this theme merging is getting out of hand + # but we want kw > BinaryDiscriminationCalibrationCurves > Scatter, so we need to somehow set things + # after the theme merging above, especially, since we also pass those to series!, + # which then again tries to merge the kw args with the theme. + set_from_kw!(scatter_theme, :makersize, kw, 5) + set_from_kw!(scatter_theme, :marker, kw, :rect) + + if ismissing(per_expert_calibration_curves) + ax = Axis(subfig; title="Detection calibration", xlabel="Expert agreement rate", + ylabel="Predicted positive probability") + else + per_expert = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :PerExpert; + solid_color=:darkgrey, color=nothing) + set_from_kw!(per_expert, :linewidth, kw, 2) + ax = series_plot!(subfig, per_expert_calibration_curves, nothing; legend=nothing, + title="Detection calibration", xlabel="Expert agreement rate", + ylabel="Predicted positive probability", scatter=scatter_theme, + per_expert...) + end + + calibration = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, + :CalibrationCurve; + solid_color=:navyblue, markerstrokewidth=0) + + set_from_kw!(calibration, :markersize, kw, 5) + set_from_kw!(calibration, :marker, kw, :rect) + set_from_kw!(calibration, :linewidth, kw, 2) + + Makie.series!(ax, calibration_curve; calibration...) + + ideal_theme = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :Ideal; + color=(:black, 0.5), + linestyle=:dash) + set_from_kw!(ideal_theme, :linewidth, kw, 2) + linesegments!(ax, [0, 1], [0, 1]; label="Ideal", ideal_theme...) + #TODO: expert agreement histogram underneath?? Maybe important... + # https://scikit-learn.org/stable/modules/calibration.html + return ax +end + +function Lighthouse.plot_confusion_matrix!(subfig::GridPosition, confusion::NumberMatrix, + class_labels::AbstractVector{String}, + normalize_by::Union{Symbol,Nothing}=nothing; + annotation_text_size=20, colormap=:Blues) + nclasses = length(class_labels) + if size(confusion) != (nclasses, nclasses) + throw(ArgumentError("Labels must match size of square confusion matrix. Found $(nclasses) labels for an $(size(confusion)) matrix")) + end + title = "Confusion Matrix" + if !isnothing(normalize_by) + normdim = get((Row=2, Column=1), normalize_by) do + throw(ArgumentError("normalize_by must be :Row, :Column, or `nothing`; found: $(normalize_by)")) + end + confusion = round.(confusion ./ sum(confusion; dims=normdim); digits=3) + title = "$(string(normalize_by))-Normalized Confusion" + end + class_indices = 1:nclasses + text_theme = get_theme(subfig, :ConfusionMatrix, :Text; fontsize=annotation_text_size) + heatmap_theme = get_theme(subfig, :ConfusionMatrix, :Heatmap; nan_color=(:black, 0.0)) + axis_theme = get_theme(subfig, :ConfusionMatrix, :Axis; xticklabelrotation=pi / 4, + titlealign=:left, title, + xlabel="Elected Class", ylabel="Predicted Class", + xticks=(class_indices, class_labels), + yticks=(class_indices, class_labels), + aspect=AxisAspect(1)) + + ax = Axis(subfig; axis_theme...) + + hidedecorations!(ax; label=false, ticklabels=false, grid=false) + ylims!(ax, nclasses + 0.5, 0.5) + tightlimits!(ax) + plot_bg_color = to_color(ax.backgroundcolor[]) + crange = isnothing(normalize_by) ? (0.0, maximum(filter(!isnan, confusion))) : + (0.0, 1.0) + nan_color = to_color(heatmap_theme.nan_color[]) + cmap = to_colormap(to_value(pop!(heatmap_theme, :colormap, colormap))) + heatmap!(ax, confusion'; colorrange=crange, colormap=cmap, nan_color=nan_color, + heatmap_theme...) + text_color = to_color(to_value(pop!(text_theme, :color, :black))) + function label_color(i, j) + c = confusion[i, j] + bg_color = if isnan(c) || ismissing(c) + Makie.Colors.alpha(nan_color) <= 0.0 ? plot_bg_color : nan_color + else + Makie.interpolated_getindex(cmap, c, crange) + end + return high_contrast(bg_color, text_color) + end + annos = vec([(string(confusion[i, j]), Point2f(j, i)) + for i in class_indices, j in class_indices]) + colors = vec([label_color(i, j) for i in class_indices, j in class_indices]) + text!(ax, annos; align=(:center, :center), color=colors, fontsize=annotation_text_size, + text_theme...) + return ax +end + +function text_attributes(values, groups, bar_colors, bg_color, text_color) + aligns = NTuple{2,Symbol}[] + offsets = NTuple{2,Int}[] + text_colors = RGBAf[] + for (i, k) in enumerate(values) + group = groups isa AbstractVector ? groups[i] : groups + bar_color = bar_colors[group] + # Plot text inside bar + if k > 0.85 + push!(aligns, (:right, :center)) + push!(offsets, (-10, 0)) + push!(text_colors, high_contrast(bar_color, text_color)) + else + # plot text next to bar + push!(aligns, (:left, :center)) + push!(offsets, (10, 0)) + push!(text_colors, high_contrast(bg_color, text_color)) + end + end + return aligns, offsets, text_colors +end + +function Lighthouse.plot_kappas!(subfig::GridPosition, per_class_kappas::NumberVector, + class_labels::AbstractVector{String}, + per_class_IRA_kappas=nothing; + color=[:lightgrey, :lightblue], annotation_text_size=20) + nclasses = length(class_labels) + axis_theme = get_theme(subfig, :Kappas, :Axis; aspect=AxisAspect(1), titlealign=:left, + xlabel="Cohen's kappa", xticks=[0, 1], yreversed=true, + yticks=(1:nclasses, class_labels)) + + text_theme = get_theme(subfig, :Kappas, :Text; fontsize=annotation_text_size) + text_color = to_color(to_value(pop!(text_theme, :color, to_color(:black)))) + bars_theme = get_theme(subfig, :Kappas, :BarPlot; color=color) + bar_colors = to_color.(bars_theme.color[]) + + ax = Axis(subfig[1, 1]; axis_theme...) + bg_color = to_color(ax.backgroundcolor[]) + xlims!(ax, 0, 1) + if !has_value(per_class_IRA_kappas) + ax.title = "Algorithm-expert agreement" + annotations = map(enumerate(per_class_kappas)) do (i, k) + return (string(round(k; digits=3)), Point2f(max(0, k), i)) + end + aligns, offsets, text_colors = text_attributes(per_class_kappas, 2, bar_colors, + bg_color, text_color) + barplot!(ax, per_class_kappas; direction=:x, color=bar_colors[2]) + text!(ax, annotations; align=aligns, offset=offsets, color=text_colors, + text_theme...) + else + ax.title = "Inter-rater reliability" + values = vcat(per_class_kappas, per_class_IRA_kappas) + groups = vcat(fill(2, nclasses), fill(1, nclasses)) + xvals = vcat(1:nclasses, 1:nclasses) + cmap = bar_colors + bars = barplot!(ax, xvals, max.(0, values); dodge=groups, color=groups, + direction=:x, + colormap=cmap) + # This is a bit hacky, but for now the easiest way to figure out the exact, dodged positions + rectangles = bars.plots[][1][] + dodged_y = last.(minimum.(rectangles)) .+ (last.(widths.(rectangles)) ./ 2) + textpos = Point2f.(max.(0, values), dodged_y) + + labels = string.(round.(values; digits=3)) + aligns, offsets, text_colors = text_attributes(values, groups, bar_colors, bg_color, + text_color) + text!(ax, labels; position=textpos, align=aligns, offset=offsets, color=text_colors, + text_theme...) + labels = ["Expert-vs-expert IRA", "Algorithm-vs-expert"] + entries = map(c -> PolyElement(; color=c, strokewidth=0, strokecolor=:white), cmap) + legend = Legend(subfig[1, 1, Bottom()], entries, labels; tellwidth=false, + tellheight=true, + labelsize=12, padding=(0, 0, 0, 0), framevisible=false, + patchsize=(10, 10), + patchlabelgap=6, labeljustification=:left) + legend.margin = (0, 0, 0, 60) + end + return ax +end + +function Lighthouse.evaluation_metrics_plot(data::Dict; kwargs...) + return evaluation_metrics_plot(EvaluationV1(data); kwargs...) +end + +function Lighthouse.evaluation_metrics_plot(row::EvaluationV1; size=(1000, 1000), + fontsize=12) + fig = Figure(; size=size, Axis=(titlesize=17,)) + + # Confusion + plot_confusion_matrix!(fig[1, 1], row.confusion_matrix, row.class_labels, + :Column; + annotation_text_size=fontsize) + plot_confusion_matrix!(fig[1, 2], row.confusion_matrix, row.class_labels, :Row; + annotation_text_size=fontsize) + # Kappas + IRA_kappa_data = nothing + multiclass = length(row.class_labels) > 2 + labels = multiclass ? vcat("Multiclass", row.class_labels) : row.class_labels + kappa_data = multiclass ? vcat(row.multiclass_kappa, row.per_class_kappas) : + row.per_class_kappas + + if issubset([:multiclass_IRA_kappas, :per_class_IRA_kappas], keys(row)) && + all(has_value.([row.multiclass_IRA_kappas, row.per_class_IRA_kappas])) + IRA_kappa_data = multiclass ? + vcat(row.multiclass_IRA_kappas, row.per_class_IRA_kappas) : + row.per_class_IRA_kappas + end + + plot_kappas!(fig[1, 3], kappa_data, labels, IRA_kappa_data; + annotation_text_size=fontsize) + + # Curves + ax = plot_roc_curves!(fig[2, 1], row.per_class_roc_curves, + row.per_class_roc_aucs, + row.class_labels; legend=nothing) + + plot_pr_curves!(fig[2, 2], row.per_class_pr_curves, row.class_labels; + legend=nothing) + + plot_reliability_calibration_curves!(fig[3, 1], + row.per_class_reliability_calibration_curves, + row.per_class_reliability_calibration_scores, + row.class_labels; legend=nothing) + + legend_pos = 2:3 + if has_value(row.discrimination_calibration_curve) + legend_pos = 3 + plot_binary_discrimination_calibration_curves!(fig[3, 2], + row.discrimination_calibration_curve, + row.discrimination_calibration_score, + row.per_expert_discrimination_calibration_curves, + row.per_expert_discrimination_calibration_scores, + row.optimal_threshold, + row.class_labels[row.optimal_threshold_class]) + end + legend_plots = filter(Makie.get_plots(ax)) do plot + return haskey(plot, :label) + end + elements = map(legend_plots) do elem + return [PolyElement(; color=elem.color, strokecolor=:transparent)] + end + + function label_str(i) + auc = round(row.per_class_roc_aucs[i]; digits=2) + mse = round(row.per_class_reliability_calibration_scores[i]; digits=2) + return ["""ROC AUC $auc + Cal. MSE $mse + """] + end + classes = row.class_labels + nclasses = length(classes) + class_labels = label_str.(1:nclasses) + Legend(fig[3, legend_pos], elements, class_labels, classes; nbanks=2, tellwidth=false, + tellheight=false, + labelsize=11, titlesize=14, titlegap=5, groupgap=6, labelhalign=:left, + labelvalign=:center) + colgap!(fig.layout, 2) + rowgap!(fig.layout, 4) + return fig +end + +# Helper to more easily define the non mutating versions +function axisplot(func, args; size=(800, 600), plot_kw...) + fig = Figure(; size=size) + ax = func(fig[1, 1], args...; plot_kw...) + # ax.plots[1] is not really that great, but there isn't a FigureAxis object right now + # this will need to wait for when we figure out a better recipe integration + return Makie.FigureAxisPlot(fig, ax, ax.scene.plots[1]) +end + +function Lighthouse.plot_confusion_matrix(args...; kw...) + return axisplot(plot_confusion_matrix!, args; + kw...) +end + +function Lighthouse.plot_reliability_calibration_curves(args...; kw...) + return axisplot(plot_reliability_calibration_curves!, args; kw...) +end + +function Lighthouse.plot_binary_discrimination_calibration_curves(args...; kw...) + return axisplot(plot_binary_discrimination_calibration_curves!, args; kw...) +end + +Lighthouse.plot_pr_curves(args...; kw...) = axisplot(plot_pr_curves!, args; kw...) + +Lighthouse.plot_roc_curves(args...; kw...) = axisplot(plot_roc_curves!, args; kw...) + +Lighthouse.plot_kappas(args...; kw...) = axisplot(plot_kappas!, args; kw...) + +##### +##### Deprecation support +##### + +function Lighthouse.evaluation_metrics_plot(predicted_hard_labels::AbstractVector, + predicted_soft_labels::AbstractMatrix, + elected_hard_labels::AbstractVector, classes, + thresholds; + votes::Union{Nothing,AbstractMatrix}=nothing, + strata::Union{Nothing, + AbstractVector{Set{T}} where T}=nothing, + optimal_threshold_class::Union{Nothing,Integer}=nothing) + Base.depwarn(""" + ``` + evaluation_metrics_plot(predicted_hard_labels::AbstractVector, + predicted_soft_labels::AbstractMatrix, + elected_hard_labels::AbstractVector, classes, thresholds; + votes::Union{Nothing,AbstractMatrix}=nothing, + strata::Union{Nothing,AbstractVector{Set{T}} where T}=nothing, + optimal_threshold_class::Union{Nothing,Integer}=nothing) + ``` + has been deprecated in favor of + ``` + plot_dict = evaluation_metrics(predicted_hard_labels, predicted_soft_labels, + elected_hard_labels, classes, thresholds; + votes, strata, optimal_threshold_class) + (evaluation_metrics_plot(plot_dict), plot_dict) + ``` + """, :evaluation_metrics_plot) + plot_dict = evaluation_metrics(predicted_hard_labels, predicted_soft_labels, + elected_hard_labels, classes, thresholds; votes, strata, + optimal_threshold_class) + return evaluation_metrics_plot(plot_dict), plot_dict +end + +end # module diff --git a/src/Lighthouse.jl b/src/Lighthouse.jl index 9154d82..31f063b 100644 --- a/src/Lighthouse.jl +++ b/src/Lighthouse.jl @@ -4,7 +4,6 @@ using Statistics, Dates, LinearAlgebra, Random, Logging using Base.Threads using StatsBase: StatsBase using TensorBoardLogger -using Makie using Printf using Legolas: Legolas, @schema, @version, lift using Tables @@ -40,4 +39,9 @@ export log_array!, log_arrays! include("deprecations.jl") +@static if !isdefined(Base, :get_extension) + include("../ext/LighthouseMakieExt.jl") + using .LighthouseMakieExt +end + end # module diff --git a/src/plotting.jl b/src/plotting.jl index 69a5d26..665dbc8 100644 --- a/src/plotting.jl +++ b/src/plotting.jl @@ -1,5 +1,3 @@ -get_parent(x::Makie.GridPosition) = x.layout.parent - # We can't rely on inference to always give us fully typed # Vector{<: Number} so we add `{T} where T` to the the mix # This makes the number like type a bit absurd, but is still nice for @@ -8,42 +6,6 @@ const NumberLike = Union{Number,Missing,Nothing,T} where {T} const NumberVector = AbstractVector{<:NumberLike} const NumberMatrix = AbstractMatrix{<:NumberLike} -##### -##### Helpers for theming and color generation...May want to move them to Colors.jl / Makie.jl -##### -using Makie.Colors: LCHab, distinguishable_colors, RGB, Colorant - -function get_theme(scene, key1::Symbol, key2::Symbol; defaults...) - return get_theme(Makie.get_scene(scene), key1, key2; defaults...) -end - -function get_theme(fig::GridPosition, key1::Symbol, key2::Symbol; defaults...) - return get_theme(get_parent(fig), key1, key2; defaults...) -end - -# This function helps us to get the theme from a scene, that we can apply to our plotting functions -function get_theme(scene::Scene, key1::Symbol, key2::Symbol; defaults...) - scene_theme = theme(scene) - sub_theme = get(scene_theme, key1, Theme()) - # The priority is key1.key2 > defaults > key2 - # this way defaults are overwritten by theme options specifically set for our theme. - # Consider Kappas.Axis for key1/2, what we want is, that if there are defaults in Kappas.Axis - # they should overwrite our generic lighthouse defaults. But anything not specified in Kappas.Axis/defaults, - # should fall back to scene_theme.Axis - return merge(get(sub_theme, key2, Theme()), Theme(; defaults...), - get(scene_theme, key2, Theme())) -end - -function high_contrast(background_color::Colorant, target_color::Colorant; - # chose from whole lightness spectrum - lchoices=range(0; stop=100, length=15)) - target = LCHab(target_color) - color = distinguishable_colors(1, [RGB(background_color)]; dropseed=true, - lchoices=lchoices, - cchoices=[target.c], hchoices=[target.h]) - return RGBAf(color[1], Makie.Colors.alpha(target_color)) -end - """ Tuple{<:NumberVector, <: NumberVector} @@ -58,377 +20,17 @@ A series of XYVectors, or a single xyvector. """ const SeriesCurves = Union{XYVector,AbstractVector{<:XYVector}} -function series_plot!(subfig::GridPosition, per_class_pr_curves::SeriesCurves, - class_labels::Union{Nothing,AbstractVector{String}}; legend=:lt, - title="No title", - xlabel="x label", ylabel="y label", solid_color=nothing, - color=nothing, - linewidth=nothing, scatter=NamedTuple()) - axis_theme = get_theme(subfig, :SeriesPlot, :Axis; title=title, titlealign=:left, - xlabel=xlabel, - ylabel=ylabel, aspect=AxisAspect(1), - xticks=0:0.2:1, yticks=0.2:0.2:1) - - ax = Axis(subfig; axis_theme...) - # Not the most elegant, but this way we can let the theming to the Series theme, or - # pass it through explicitely - series_theme = get_theme(subfig, :SeriesPlot, :Series; scatter...) - isnothing(solid_color) || (series_theme[:solid_color] = solid_color) - isnothing(color) || (series_theme[:color] = color) - isnothing(linewidth) || (series_theme[:linewidth] = linewidth) - series_theme = merge(series_theme, Attributes(; scatter...)) - hidedecorations!(ax; label=false, ticklabels=false, grid=false) - limits!(ax, 0, 1, 0, 1) - Makie.series!(ax, per_class_pr_curves; labels=class_labels, series_theme...) - if !isnothing(legend) - axislegend(ax; position=legend) - end - return ax -end - -function plot_pr_curves!(subfig::GridPosition, per_class_pr_curves::SeriesCurves, - class_labels::Union{Nothing,AbstractVector{String}}; legend=:lt, - title="PR curves", - xlabel="True positive rate", ylabel="Precision", - scatter=NamedTuple(), - solid_color=nothing) - return series_plot!(subfig, per_class_pr_curves, class_labels; legend=legend, - title=title, xlabel=xlabel, - ylabel=ylabel, scatter=scatter, solid_color=solid_color) -end - -function plot_roc_curves!(subfig::GridPosition, per_class_roc_curves::SeriesCurves, - per_class_roc_aucs::NumberVector, - class_labels::AbstractVector{<:String}; - legend=:rb, title="ROC curves", xlabel="False positive rate", - ylabel="True positive rate") - auc_labels = [@sprintf("%s (AUC: %.3f)", class, per_class_roc_aucs[i]) - for (i, class) in enumerate(class_labels)] - - return series_plot!(subfig, per_class_roc_curves, auc_labels; legend=legend, - title=title, xlabel=xlabel, - ylabel=ylabel) -end - -function plot_reliability_calibration_curves!(subfig::GridPosition, - per_class_reliability_calibration_curves::SeriesCurves, - per_class_reliability_calibration_scores::NumberVector, - class_labels::AbstractVector{String}; - legend=:rb) - calibration_score_labels = map(enumerate(class_labels)) do (i, class) - @sprintf("%s (MSE: %.3f)", class, per_class_reliability_calibration_scores[i]) - end - - scatter_theme = get_theme(subfig, :ReliabilityCalibrationCurves, :Scatter; - marker=Circle, - markersize=5, strokewidth=0) - ideal_theme = get_theme(subfig, :ReliabilityCalibrationCurves, :Ideal; - color=(:black, 0.5), - linestyle=:dash, linewidth=2) - - ax = series_plot!(subfig, per_class_reliability_calibration_curves, - calibration_score_labels; - legend=legend, title="Prediction reliability calibration", - xlabel="Predicted probability bin", ylabel="Fraction of positives", - scatter=scatter_theme) - #TODO: mean predicted value histogram underneath?? Maybe important... - # https://scikit-learn.org/stable/modules/calibration.html - linesegments!(ax, [0, 1], [0, 1]; ideal_theme...) - return ax -end - -function set_from_kw!(theme, key, kw, default) - if haskey(kw, key) - theme[key] = getproperty(kw, key) - elseif !haskey(theme, key) - theme[key] = default - end - return -end - -function plot_binary_discrimination_calibration_curves!(subfig::GridPosition, - calibration_curve::SeriesCurves, - calibration_score, - per_expert_calibration_curves::Union{SeriesCurves, - Missing}, - per_expert_calibration_scores, - optimal_threshold, - discrimination_class::AbstractString; - kw...) - kw = values(kw) - scatter_theme = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :Scatter; - strokewidth=0) - # Hayaah, this theme merging is getting out of hand - # but we want kw > BinaryDiscriminationCalibrationCurves > Scatter, so we need to somehow set things - # after the theme merging above, especially, since we also pass those to series!, - # which then again tries to merge the kw args with the theme. - set_from_kw!(scatter_theme, :makersize, kw, 5) - set_from_kw!(scatter_theme, :marker, kw, :rect) - - if ismissing(per_expert_calibration_curves) - ax = Axis(subfig; title="Detection calibration", xlabel="Expert agreement rate", - ylabel="Predicted positive probability") - else - per_expert = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :PerExpert; - solid_color=:darkgrey, color=nothing) - set_from_kw!(per_expert, :linewidth, kw, 2) - ax = series_plot!(subfig, per_expert_calibration_curves, nothing; legend=nothing, - title="Detection calibration", xlabel="Expert agreement rate", - ylabel="Predicted positive probability", scatter=scatter_theme, - per_expert...) - end - - calibration = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, - :CalibrationCurve; - solid_color=:navyblue, markerstrokewidth=0) - - set_from_kw!(calibration, :markersize, kw, 5) - set_from_kw!(calibration, :marker, kw, :rect) - set_from_kw!(calibration, :linewidth, kw, 2) - - Makie.series!(ax, calibration_curve; calibration...) - - ideal_theme = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :Ideal; - color=(:black, 0.5), - linestyle=:dash) - set_from_kw!(ideal_theme, :linewidth, kw, 2) - linesegments!(ax, [0, 1], [0, 1]; label="Ideal", ideal_theme...) - #TODO: expert agreement histogram underneath?? Maybe important... - # https://scikit-learn.org/stable/modules/calibration.html - return ax -end - -function plot_confusion_matrix!(subfig::GridPosition, confusion::NumberMatrix, - class_labels::AbstractVector{String}, - normalize_by::Union{Symbol,Nothing}=nothing; - annotation_text_size=20, colormap=:Blues) - nclasses = length(class_labels) - if size(confusion) != (nclasses, nclasses) - throw(ArgumentError("Labels must match size of square confusion matrix. Found $(nclasses) labels for an $(size(confusion)) matrix")) - end - title = "Confusion Matrix" - if !isnothing(normalize_by) - normdim = get((Row=2, Column=1), normalize_by) do - throw(ArgumentError("normalize_by must be :Row, :Column, or `nothing`; found: $(normalize_by)")) - end - confusion = round.(confusion ./ sum(confusion; dims=normdim); digits=3) - title = "$(string(normalize_by))-Normalized Confusion" - end - class_indices = 1:nclasses - text_theme = get_theme(subfig, :ConfusionMatrix, :Text; fontsize=annotation_text_size) - heatmap_theme = get_theme(subfig, :ConfusionMatrix, :Heatmap; nan_color=(:black, 0.0)) - axis_theme = get_theme(subfig, :ConfusionMatrix, :Axis; xticklabelrotation=pi / 4, - titlealign=:left, title, - xlabel="Elected Class", ylabel="Predicted Class", - xticks=(class_indices, class_labels), - yticks=(class_indices, class_labels), - aspect=AxisAspect(1)) - - ax = Axis(subfig; axis_theme...) - - hidedecorations!(ax; label=false, ticklabels=false, grid=false) - ylims!(ax, nclasses + 0.5, 0.5) - tightlimits!(ax) - plot_bg_color = to_color(ax.backgroundcolor[]) - crange = isnothing(normalize_by) ? (0.0, maximum(filter(!isnan, confusion))) : - (0.0, 1.0) - nan_color = to_color(heatmap_theme.nan_color[]) - cmap = to_colormap(to_value(pop!(heatmap_theme, :colormap, colormap))) - heatmap!(ax, confusion'; colorrange=crange, colormap=cmap, nan_color=nan_color, - heatmap_theme...) - text_color = to_color(to_value(pop!(text_theme, :color, :black))) - function label_color(i, j) - c = confusion[i, j] - bg_color = if isnan(c) || ismissing(c) - Makie.Colors.alpha(nan_color) <= 0.0 ? plot_bg_color : nan_color - else - Makie.interpolated_getindex(cmap, c, crange) - end - return high_contrast(bg_color, text_color) - end - annos = vec([(string(confusion[i, j]), Point2f(j, i)) - for i in class_indices, j in class_indices]) - colors = vec([label_color(i, j) for i in class_indices, j in class_indices]) - text!(ax, annos; align=(:center, :center), color=colors, fontsize=annotation_text_size, - text_theme...) - return ax -end - -function text_attributes(values, groups, bar_colors, bg_color, text_color) - aligns = NTuple{2,Symbol}[] - offsets = NTuple{2,Int}[] - text_colors = RGBAf[] - for (i, k) in enumerate(values) - group = groups isa AbstractVector ? groups[i] : groups - bar_color = bar_colors[group] - # Plot text inside bar - if k > 0.85 - push!(aligns, (:right, :center)) - push!(offsets, (-10, 0)) - push!(text_colors, high_contrast(bar_color, text_color)) - else - # plot text next to bar - push!(aligns, (:left, :center)) - push!(offsets, (10, 0)) - push!(text_colors, high_contrast(bg_color, text_color)) - end - end - return aligns, offsets, text_colors -end - -function plot_kappas!(subfig::GridPosition, per_class_kappas::NumberVector, - class_labels::AbstractVector{String}, per_class_IRA_kappas=nothing; - color=[:lightgrey, :lightblue], annotation_text_size=20) - nclasses = length(class_labels) - axis_theme = get_theme(subfig, :Kappas, :Axis; aspect=AxisAspect(1), titlealign=:left, - xlabel="Cohen's kappa", xticks=[0, 1], yreversed=true, - yticks=(1:nclasses, class_labels)) - - text_theme = get_theme(subfig, :Kappas, :Text; fontsize=annotation_text_size) - text_color = to_color(to_value(pop!(text_theme, :color, to_color(:black)))) - bars_theme = get_theme(subfig, :Kappas, :BarPlot; color=color) - bar_colors = to_color.(bars_theme.color[]) - - ax = Axis(subfig[1, 1]; axis_theme...) - bg_color = to_color(ax.backgroundcolor[]) - xlims!(ax, 0, 1) - if !has_value(per_class_IRA_kappas) - ax.title = "Algorithm-expert agreement" - annotations = map(enumerate(per_class_kappas)) do (i, k) - return (string(round(k; digits=3)), Point2f(max(0, k), i)) - end - aligns, offsets, text_colors = text_attributes(per_class_kappas, 2, bar_colors, - bg_color, text_color) - barplot!(ax, per_class_kappas; direction=:x, color=bar_colors[2]) - text!(ax, annotations; align=aligns, offset=offsets, color=text_colors, - text_theme...) - else - ax.title = "Inter-rater reliability" - values = vcat(per_class_kappas, per_class_IRA_kappas) - groups = vcat(fill(2, nclasses), fill(1, nclasses)) - xvals = vcat(1:nclasses, 1:nclasses) - cmap = bar_colors - bars = barplot!(ax, xvals, max.(0, values); dodge=groups, color=groups, - direction=:x, - colormap=cmap) - # This is a bit hacky, but for now the easiest way to figure out the exact, dodged positions - rectangles = bars.plots[][1][] - dodged_y = last.(minimum.(rectangles)) .+ (last.(widths.(rectangles)) ./ 2) - textpos = Point2f.(max.(0, values), dodged_y) - - labels = string.(round.(values; digits=3)) - aligns, offsets, text_colors = text_attributes(values, groups, bar_colors, bg_color, - text_color) - text!(ax, labels; position=textpos, align=aligns, offset=offsets, color=text_colors, - text_theme...) - labels = ["Expert-vs-expert IRA", "Algorithm-vs-expert"] - entries = map(c -> PolyElement(; color=c, strokewidth=0, strokecolor=:white), cmap) - legend = Legend(subfig[1, 1, Bottom()], entries, labels; tellwidth=false, - tellheight=true, - labelsize=12, padding=(0, 0, 0, 0), framevisible=false, - patchsize=(10, 10), - patchlabelgap=6, labeljustification=:left) - legend.margin = (0, 0, 0, 60) - end - return ax -end - """ evaluation_metrics_plot(data::Dict; size=(1000, 1000), fontsize=12) evaluation_metrics_plot(row::EvaluationV1; kwargs...) Plot all evaluation metrics generated via [`evaluation_metrics_record`](@ref) and/or [`evaluation_metrics`](@ref) in a single image. + +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -function evaluation_metrics_plot(data::Dict; kwargs...) - return evaluation_metrics_plot(EvaluationV1(data); kwargs...) -end - -function evaluation_metrics_plot(row::EvaluationV1; size=(1000, 1000), - fontsize=12) - fig = Figure(; size=size, Axis=(titlesize=17,)) - - # Confusion - plot_confusion_matrix!(fig[1, 1], row.confusion_matrix, row.class_labels, - :Column; - annotation_text_size=fontsize) - plot_confusion_matrix!(fig[1, 2], row.confusion_matrix, row.class_labels, :Row; - annotation_text_size=fontsize) - # Kappas - IRA_kappa_data = nothing - multiclass = length(row.class_labels) > 2 - labels = multiclass ? vcat("Multiclass", row.class_labels) : row.class_labels - kappa_data = multiclass ? vcat(row.multiclass_kappa, row.per_class_kappas) : - row.per_class_kappas - - if issubset([:multiclass_IRA_kappas, :per_class_IRA_kappas], keys(row)) && - all(has_value.([row.multiclass_IRA_kappas, row.per_class_IRA_kappas])) - IRA_kappa_data = multiclass ? - vcat(row.multiclass_IRA_kappas, row.per_class_IRA_kappas) : - row.per_class_IRA_kappas - end - - plot_kappas!(fig[1, 3], kappa_data, labels, IRA_kappa_data; - annotation_text_size=fontsize) - - # Curves - ax = plot_roc_curves!(fig[2, 1], row.per_class_roc_curves, - row.per_class_roc_aucs, - row.class_labels; legend=nothing) - - plot_pr_curves!(fig[2, 2], row.per_class_pr_curves, row.class_labels; - legend=nothing) - - plot_reliability_calibration_curves!(fig[3, 1], - row.per_class_reliability_calibration_curves, - row.per_class_reliability_calibration_scores, - row.class_labels; legend=nothing) - - legend_pos = 2:3 - if has_value(row.discrimination_calibration_curve) - legend_pos = 3 - plot_binary_discrimination_calibration_curves!(fig[3, 2], - row.discrimination_calibration_curve, - row.discrimination_calibration_score, - row.per_expert_discrimination_calibration_curves, - row.per_expert_discrimination_calibration_scores, - row.optimal_threshold, - row.class_labels[row.optimal_threshold_class]) - end - legend_plots = filter(Makie.get_plots(ax)) do plot - return haskey(plot, :label) - end - elements = map(legend_plots) do elem - return [PolyElement(; color=elem.color, strokecolor=:transparent)] - end - - function label_str(i) - auc = round(row.per_class_roc_aucs[i]; digits=2) - mse = round(row.per_class_reliability_calibration_scores[i]; digits=2) - return ["""ROC AUC $auc - Cal. MSE $mse - """] - end - classes = row.class_labels - nclasses = length(classes) - class_labels = label_str.(1:nclasses) - Legend(fig[3, legend_pos], elements, class_labels, classes; nbanks=2, tellwidth=false, - tellheight=false, - labelsize=11, titlesize=14, titlegap=5, groupgap=6, labelhalign=:left, - labelvalign=:center) - colgap!(fig.layout, 2) - rowgap!(fig.layout, 4) - return fig -end - -# Helper to more easily define the non mutating versions -function axisplot(func, args; size=(800, 600), plot_kw...) - fig = Figure(; size=size) - ax = func(fig[1, 1], args...; plot_kw...) - # ax.plots[1] is not really that great, but there isn't a FigureAxis object right now - # this will need to wait for when we figure out a better recipe integration - return Makie.FigureAxisPlot(fig, ax, ax.scene.plots[1]) -end +function evaluation_metrics_plot end """ plot_confusion_matrix!(subfig::GridPosition, args...; kw...) @@ -452,8 +54,12 @@ fig = Figure() ax = plot_confusion_matrix!(fig[1, 1], rand(2, 2), ["1", "2"], :Row) ax = plot_confusion_matrix!(fig[1, 2], rand(2, 2), ["1", "2"], :Column) ``` + +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -plot_confusion_matrix(args...; kw...) = axisplot(plot_confusion_matrix!, args; kw...) +function plot_confusion_matrix end +function plot_confusion_matrix! end """ plot_reliability_calibration_curves!(fig::SubFigure, args...; kw...) @@ -463,10 +69,11 @@ plot_confusion_matrix(args...; kw...) = axisplot(plot_confusion_matrix!, args; k class_labels::AbstractVector{String}; legend=:rb, size=(800, 600)) +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -function plot_reliability_calibration_curves(args...; kw...) - return axisplot(plot_reliability_calibration_curves!, args; kw...) -end +function plot_reliability_calibration_curves end +function plot_reliability_calibration_curves! end """ plot_binary_discrimination_calibration_curves!(fig::SubFigure, args...; kw...) @@ -477,10 +84,11 @@ end discrimination_class::AbstractString; marker=:rect, markersize=5, linewidth=2) +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -function plot_binary_discrimination_calibration_curves(args...; kw...) - return axisplot(plot_binary_discrimination_calibration_curves!, args; kw...) -end +function plot_binary_discrimination_calibration_curves end +function plot_binary_discrimination_calibration_curves! end """ plot_pr_curves!(subfig::GridPosition, args...; kw...) @@ -494,8 +102,12 @@ end - `scatter::Union{Nothing, NamedTuple}`: can be set to a named tuples of attributes that are forwarded to the scatter call (e.g. markersize). If nothing, no scatter is added. + +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -plot_pr_curves(args...; kw...) = axisplot(plot_pr_curves!, args; kw...) +function plot_pr_curves end +function plot_pr_curves! end """ plot_roc_curves!(subfig::GridPosition, args...; kw...) @@ -512,8 +124,11 @@ plot_pr_curves(args...; kw...) = axisplot(plot_pr_curves!, args; kw...) - `scatter::Union{Nothing, NamedTuple}`: can be set to a named tuples of attributes that are forwarded to the scatter call (e.g. markersize). If nothing, no scatter is added. +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -plot_roc_curves(args...; kw...) = axisplot(plot_roc_curves!, args; kw...) +function plot_roc_curves end +function plot_roc_curves! end """ plot_kappas!(subfig::GridPosition, args...; kw...) @@ -523,8 +138,12 @@ plot_roc_curves(args...; kw...) = axisplot(plot_roc_curves!, args; kw...) per_class_IRA_kappas=nothing; size=(800, 600), annotation_text_size=20) + +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -plot_kappas(args...; kw...) = axisplot(plot_kappas!, args; kw...) +function plot_kappas end +function plot_kappas! end ##### ##### Deprecation support @@ -548,32 +167,8 @@ See [`evaluation_metrics`](@ref) for a description of the arguments. This method is deprecated in favor of calling `evaluation_metrics` and [`evaluation_metrics_plot`](@ref) separately. + +!!! note + This function requires a valid Makie backend (e.g. CairoMakie) to be loaded. """ -function evaluation_metrics_plot(predicted_hard_labels::AbstractVector, - predicted_soft_labels::AbstractMatrix, - elected_hard_labels::AbstractVector, classes, thresholds; - votes::Union{Nothing,AbstractMatrix}=nothing, - strata::Union{Nothing,AbstractVector{Set{T}} where T}=nothing, - optimal_threshold_class::Union{Nothing,Integer}=nothing) - Base.depwarn(""" - ``` - evaluation_metrics_plot(predicted_hard_labels::AbstractVector, - predicted_soft_labels::AbstractMatrix, - elected_hard_labels::AbstractVector, classes, thresholds; - votes::Union{Nothing,AbstractMatrix}=nothing, - strata::Union{Nothing,AbstractVector{Set{T}} where T}=nothing, - optimal_threshold_class::Union{Nothing,Integer}=nothing) - ``` - has been deprecated in favor of - ``` - plot_dict = evaluation_metrics(predicted_hard_labels, predicted_soft_labels, - elected_hard_labels, classes, thresholds; - votes, strata, optimal_threshold_class) - (evaluation_metrics_plot(plot_dict), plot_dict) - ``` - """, :evaluation_metrics_plot) - plot_dict = evaluation_metrics(predicted_hard_labels, predicted_soft_labels, - elected_hard_labels, classes, thresholds; votes, strata, - optimal_threshold_class) - return evaluation_metrics_plot(plot_dict), plot_dict -end +function evaluation_metrics_plot end diff --git a/src/row.jl b/src/row.jl index 4e1fbd0..ca583a0 100644 --- a/src/row.jl +++ b/src/row.jl @@ -148,7 +148,6 @@ end function _observation_table_to_inputs(observation_table) Legolas.validate(Tables.schema(observation_table), ObservationV1SchemaVersion()) df_table = Tables.columns(observation_table) - votes = missing if any(ismissing, df_table.votes) && !all(ismissing, df_table.votes) throw(ArgumentError("`:votes` must either be all `missing` or contain no `missing`")) end diff --git a/test/row.jl b/test/row.jl index 94d9158..d5d94da 100644 --- a/test/row.jl +++ b/test/row.jl @@ -95,7 +95,12 @@ end transform!(df_table, :votes => ByRow(v -> ismissing(v) ? [1, 2, 3] : v); renamecols=false) - @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table) + @static if VERSION < v"1.10" + # the error type changed between Julia versions! + @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table) + else + @test_throws DimensionMismatch Lighthouse._observation_table_to_inputs(df_table) + end @test_throws DimensionMismatch Lighthouse._inputs_to_observation_table(; predicted_soft_labels,