From fb1290825e3f27249ec2238a558a0b3da1e293f4 Mon Sep 17 00:00:00 2001 From: duncanpo Date: Tue, 22 Oct 2024 12:46:40 -0400 Subject: [PATCH] add tests for AutoTrace --- .../example1/example1_trycatch.m | 12 ++ test/tautotrace.m | 197 +++++++++++++++++- 2 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 test/autotrace_examples/example1/example1_trycatch.m diff --git a/test/autotrace_examples/example1/example1_trycatch.m b/test/autotrace_examples/example1/example1_trycatch.m new file mode 100644 index 0000000..2d58fc1 --- /dev/null +++ b/test/autotrace_examples/example1/example1_trycatch.m @@ -0,0 +1,12 @@ +function yf = example1_trycatch(at, n) +% example code for testing auto instrumentation. This example should not +% use beginTrace method and instead should be called directly. + +% Copyright 2024 The MathWorks, Inc. + +try + [x, y] = generate_data(n); + yf = best_fit_line(x,y); +catch ME + handleError(at, ME); +end \ No newline at end of file diff --git a/test/tautotrace.m b/test/tautotrace.m index 3873693..e59d3fc 100644 --- a/test/tautotrace.m +++ b/test/tautotrace.m @@ -21,10 +21,14 @@ function setupOnce(testCase) % add the utils folder to the path utilsfolder = fullfile(fileparts(mfilename('fullpath')), "utils"); testCase.applyFixture(matlab.unittest.fixtures.PathFixture(utilsfolder)); - % add the example folder to the path + % add the example folders to the path example1folder = fullfile(fileparts(mfilename('fullpath')), "autotrace_examples", "example1"); testCase.applyFixture(matlab.unittest.fixtures.PathFixture(example1folder)); commonSetupOnce(testCase); + + % configure the global tracer provider + tp = opentelemetry.sdk.trace.TracerProvider(); + setTracerProvider(tp); end end @@ -43,12 +47,7 @@ function teardown(testCase) methods (Test) function testBasic(testCase) % testBasic: instrument a simple example - - % configure the global tracer provider - tp = opentelemetry.sdk.trace.TracerProvider(); - setTracerProvider(tp); - clear("tp"); - + % set up AutoTrace at = opentelemetry.autoinstrument.AutoTrace(@example1); @@ -59,11 +58,191 @@ function testBasic(testCase) results = readJsonResults(testCase); verifyNumElements(testCase, results, 3); - % check logger name, log body and severity, trace and span IDs - verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.scope.name), "AutoTrace"); + % check tracer and span names + verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.scope.name), "AutoTrace"); % default name verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.spans.name), "generate_data"); verifyEqual(testCase, string(results{2}.resourceSpans.scopeSpans.spans.name), "best_fit_line"); verifyEqual(testCase, string(results{3}.resourceSpans.scopeSpans.spans.name), "example1"); + + % check they belong to the same trace + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.traceId, results{2}.resourceSpans.scopeSpans.spans.traceId); + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.traceId, results{3}.resourceSpans.scopeSpans.spans.traceId); + + % check parent children relationship + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.parentSpanId, results{3}.resourceSpans.scopeSpans.spans.spanId); + verifyEqual(testCase, results{2}.resourceSpans.scopeSpans.spans.parentSpanId, results{3}.resourceSpans.scopeSpans.spans.spanId); + end + + function testIncludeExcludeFiles(testCase) + % testIncludeExcludeFiles: AdditionalFiles and ExcludeFiles options + + % set up AutoTrace + at = opentelemetry.autoinstrument.AutoTrace(@example1, ... + "AdditionalFiles", "polyfit", "ExcludeFiles", "generate_data"); + + % run the example + [~] = beginTrace(at, 100); + + % perform test comparisons + results = readJsonResults(testCase); + verifyNumElements(testCase, results, 3); + + % check span names + verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.spans.name), "polyfit"); + verifyEqual(testCase, string(results{2}.resourceSpans.scopeSpans.spans.name), "best_fit_line"); + verifyEqual(testCase, string(results{3}.resourceSpans.scopeSpans.spans.name), "example1"); + + % check parent children relationship + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.parentSpanId, results{2}.resourceSpans.scopeSpans.spans.spanId); + verifyEqual(testCase, results{2}.resourceSpans.scopeSpans.spans.parentSpanId, results{3}.resourceSpans.scopeSpans.spans.spanId); + end + + function testDisableFileDetection(testCase) + % testDisableFileDetection: AutoDetectFiles set to false + + % set up AutoTrace + at = opentelemetry.autoinstrument.AutoTrace(@example1, ... + "AutoDetectFiles", false); + + % run the example + [~] = beginTrace(at, 100); + + % perform test comparisons + results = readJsonResults(testCase); + + % should only be 1 span + verifyNumElements(testCase, results, 1); + verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.spans.name), "example1"); + end + + function testNonFileOptions(testCase) + % testNonFileOptions: other options not related to files, + % "TracerName", "TracerVersion", "TracerSchema", "Attributes", + % "SpanKind" + + tracername = "foo"; + tracerversion = "1.1"; + tracerschema = "https://opentelemetry.io/schemas/1.28.0"; + spankind = "consumer"; + attrnames = ["foo" "bar"]; + attrvalues = [1 2]; + attrs = dictionary(attrnames, attrvalues); + % set up AutoTrace + at = opentelemetry.autoinstrument.AutoTrace(@example1, ... + "TracerName", tracername, "TracerVersion", tracerversion, ... + "TracerSchema", tracerschema, "SpanKind", spankind, "Attributes", attrs); + + % run the example + [~] = beginTrace(at, 100); + + % perform test comparisons + results = readJsonResults(testCase); + verifyNumElements(testCase, results, 3); + + % check specified options in each span + for i = 1:numel(results) + verifyEqual(testCase, string(results{i}.resourceSpans.scopeSpans.scope.name), tracername); + verifyEqual(testCase, string(results{i}.resourceSpans.scopeSpans.scope.version), tracerversion); + verifyEqual(testCase, string(results{i}.resourceSpans.scopeSpans.schemaUrl), tracerschema); + verifyEqual(testCase, results{i}.resourceSpans.scopeSpans.spans.kind, 5); % SpanKind consumer + + % attributes + attrkeys = string({results{i}.resourceSpans.scopeSpans.spans.attributes.key}); + + for ii = 1:numel(attrnames) + attrnameii = attrnames(ii); + idxii = find(attrkeys == attrnameii); + verifyNotEmpty(testCase, idxii); + verifyEqual(testCase, results{i}.resourceSpans.scopeSpans.spans.attributes(idxii).value.doubleValue, ... + attrvalues(ii)); + end + end + end + + function testError(testCase) + % testError: handling error situation + + % set up AutoTrace + at = opentelemetry.autoinstrument.AutoTrace(@example1); + + % run the example with an invalid input, check for error + verifyError(testCase, @()beginTrace(at, "invalid"), "MATLAB:colon:inputsMustBeNumericCharLogical"); + + % perform test comparisons + results = readJsonResults(testCase); + verifyNumElements(testCase, results, 2); + + % check span names + verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.spans.name), "generate_data"); + verifyEqual(testCase, string(results{2}.resourceSpans.scopeSpans.spans.name), "example1"); + + % check parent children relationship + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.parentSpanId, results{2}.resourceSpans.scopeSpans.spans.spanId); + + % check error status + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.status.code, 2); % error + verifyEmpty(testCase, fieldnames(results{2}.resourceSpans.scopeSpans.spans.status)); % ok, no error + end + + function testHandleError(testCase) + % testHandleError: directly call handleError method rather than using + % beginTrace method. This test should use example1_trycatch, which + % wraps a try-catch in the input function and calls handleError + % in the catch block. + + % set up AutoTrace, using example1_trycatch + at = opentelemetry.autoinstrument.AutoTrace(@example1_trycatch); + + % call example directly instead of calling beginTrace, and pass + % in an invalid input + verifyError(testCase, @()example1_trycatch(at, "invalid"), "MATLAB:colon:inputsMustBeNumericCharLogical"); + + % perform test comparisons + results = readJsonResults(testCase); + verifyNumElements(testCase, results, 2); + + % check span names + verifyEqual(testCase, string(results{1}.resourceSpans.scopeSpans.spans.name), "generate_data"); + verifyEqual(testCase, string(results{2}.resourceSpans.scopeSpans.spans.name), "example1_trycatch"); + + % check parent children relationship + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.parentSpanId, results{2}.resourceSpans.scopeSpans.spans.spanId); + + % check error status + verifyEqual(testCase, results{1}.resourceSpans.scopeSpans.spans.status.code, 2); % error + verifyEmpty(testCase, fieldnames(results{2}.resourceSpans.scopeSpans.spans.status)); % ok, no error + end + + function testMultipleInstances(testCase) + % testMultipleInstances: multiple overlapped instances should + % return an error + + % set up AutoTrace + at = opentelemetry.autoinstrument.AutoTrace(@example1); %#ok + + % set up another identical instance, check for error + verifyError(testCase, @()opentelemetry.autoinstrument.AutoTrace(@example1), "opentelemetry:autoinstrument:AutoTrace:OverlappedInstances"); + end + + function testClearInstance(~) + % testClearInstance: clear an instance and recreate a new instance + + % create and instance and then clear + at = opentelemetry.autoinstrument.AutoTrace(@example1); %#ok + clear("at") + + % create a new instance should not result in any error + at = opentelemetry.autoinstrument.AutoTrace(@example1); %#ok + end + + function testInvalidInputFunction(testCase) + % testInvalidInputFunction: negative test for invalid input + + % anonymous function + verifyError(testCase, @()opentelemetry.autoinstrument.AutoTrace(@()example1), "opentelemetry:autoinstrument:AutoTrace:AnonymousFunction"); + + % builtin function + verifyError(testCase, @()opentelemetry.autoinstrument.AutoTrace(@uplus), "opentelemetry:autoinstrument:AutoTrace:InvalidMFile"); end end end \ No newline at end of file