From f2627b935d167b59bfae4f0f9368b703c8cccb2c Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Fri, 3 Feb 2023 13:53:58 +0300 Subject: [PATCH 01/11] [CircleGraph] Add visualization draft This commit adds weight visualization example. ONE-vscode-DCO-1.0-Signed-off-by: Stanislav Ponkrashov --- media/CircleGraph/view-sidebar.js | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/media/CircleGraph/view-sidebar.js b/media/CircleGraph/view-sidebar.js index 0d048c9c..ea620cd0 100644 --- a/media/CircleGraph/view-sidebar.js +++ b/media/CircleGraph/view-sidebar.js @@ -625,6 +625,96 @@ sidebar.ParameterView = class { } }; +function getTensorShape(array) { + let shape = []; + let curArray = array; + while (Array.isArray(curArray)) { + shape.push(curArray.length); + curArray = curArray[0]; + } + return shape; +} + +function getTensorValue(tensor, index) { + let value = tensor; + for (const i of index) + {value = value[i];} + return value; +} + +function normalizeTensor(tensor, axis1, axis2, values = {}) { + let shape = getTensorShape(tensor); + let height = shape[axis1]; + let width = shape[axis2]; + let imageData = []; + + let index = Array(shape.length); + index.fill(0); + + // find min and max values in the tensor + let minValue = Infinity; + let maxValue = -Infinity; + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + index[axis1] = i; + index[axis2] = j; + let value = getTensorValue(tensor, index); + minValue = Math.min(minValue, value); + maxValue = Math.max(maxValue, value); + } + } + + // normalize values and create imageData + for (let i = 0; i < height; i++) { + let row = []; + for (let j = 0; j < width; j++) { + index[axis1] = i; + index[axis2] = j; + let value = getTensorValue(tensor, index); + row.push((value - minValue) / (maxValue - minValue)); + } + imageData.push(row); + } + return imageData; +} + +function tensorToImage(tensor, axis1, axis2, document, values = {}) { + let scale = 12; + let imageData = normalizeTensor(tensor, axis1, axis2, values); + let height = imageData.length; + let width = imageData[0].length; + let canvas = document.createElement('canvas'); + canvas.width = width * scale; + canvas.height = height * scale; + let ctx = canvas.getContext('2d'); + ctx.imageSmoothingEnabled = false; + let imageDataArray = new Uint8ClampedArray(height * width * 4); + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + let value = imageData[i][j]; + let index = (i * width + j) * 4; + + value *= 2; + let blue = Math.round(Math.max(0, 255 * (1.0 - value))); + let red = Math.round(Math.max(0, 255 * (value - 1.0))); + let green = 255 - blue - red; + + imageDataArray[index] = red; + imageDataArray[index + 1] = green; + imageDataArray[index + 2] = blue; + imageDataArray[index + 3] = 255; + } + } + let imageDataImage = new ImageData(imageDataArray, width, height); + ctx.putImageData(imageDataImage, 0, 0); + + //scale + ctx.scale(scale, scale); + ctx.drawImage(canvas, 0, 0); + + return canvas; +} + sidebar.ArgumentView = class { constructor(host, argument) { this._host = host; @@ -782,6 +872,16 @@ sidebar.ArgumentView = class { valueLine.className = "sidebar-view-item-value-line-border"; contentLine.innerHTML = state || initializer.toString(); + if (!state) { + try { + const imageLine = this._host.document.createElement("div"); + imageLine.className = "sidebar-view-item-value-line-border"; + imageLine.appendChild(tensorToImage(JSON.parse(initializer.toString()), 2, 3, this._host.document)); + this._element.appendChild(imageLine); + } catch (err) { + // do nothing + } + } } catch (err) { contentLine.innerHTML = err.toString(); this._raise("error", err); From 82bb3907d07d0833f287287af55fe3b572a8f9d7 Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Mon, 6 Feb 2023 13:16:15 +0300 Subject: [PATCH 02/11] Add axes selector --- media/CircleGraph/view-sidebar.js | 75 ++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 11 deletions(-) diff --git a/media/CircleGraph/view-sidebar.js b/media/CircleGraph/view-sidebar.js index ea620cd0..8b334088 100644 --- a/media/CircleGraph/view-sidebar.js +++ b/media/CircleGraph/view-sidebar.js @@ -637,8 +637,9 @@ function getTensorShape(array) { function getTensorValue(tensor, index) { let value = tensor; - for (const i of index) - {value = value[i];} + for (const i of index) { + value = value[i]; + } return value; } @@ -708,13 +709,72 @@ function tensorToImage(tensor, axis1, axis2, document, values = {}) { let imageDataImage = new ImageData(imageDataArray, width, height); ctx.putImageData(imageDataImage, 0, 0); - //scale + // scale ctx.scale(scale, scale); ctx.drawImage(canvas, 0, 0); return canvas; } +sidebar.VisualTensorView = class { + constructor(host, tensor) { + this._host = host; + this._tensor = tensor; + + this._element = this._host.document.createElement("div"); + this._element.class = "sidebar-view-item-value-line-border"; + + this._tensorShape = getTensorShape(this._tensor); + if (this._tensorShape.length < 2) { + return this; + } + this._axes = [this._tensorShape.length - 2, this._tensorShape.length - 1]; + // add axes selection checkboxes + if (this._tensorShape.length > 2) { + this._checkboxes = []; + for (let i = 0; i < this._tensorShape.length; ++i) { + let checkbox = this._host.document.createElement("input"); + checkbox.type = "checkbox"; + checkbox.checked = this._axes.includes(i); + checkbox.addEventListener("change", () => { + if (checkbox.checked) { + if (this._axes.length >= 2) { + this._checkboxes[this._axes[0]].checked = false; + this._axes.shift(); + } + this._axes.push(i); + } else { + this._axes.splice(this._axes.indexOf(i), 1); + } + this.updateImage(); + }); + this._checkboxes.push(checkbox); + this._element.appendChild(checkbox); + } + } + this._element.appendChild(this._host.document.createElement("br")); + + this._image = null; + this.updateImage(); + } + + updateImage() { + if (this._image) { + this._element.removeChild(this._image); + } + this._image = null; + if (this._axes.length < 2) { + return; + } + try { + this._image = tensorToImage(this._tensor, this._axes[0], this._axes[1], this._host.document); + this._element.appendChild(this._image); + } catch (err) { + // do nothing + } + } +}; + sidebar.ArgumentView = class { constructor(host, argument) { this._host = host; @@ -873,14 +933,7 @@ sidebar.ArgumentView = class { valueLine.className = "sidebar-view-item-value-line-border"; contentLine.innerHTML = state || initializer.toString(); if (!state) { - try { - const imageLine = this._host.document.createElement("div"); - imageLine.className = "sidebar-view-item-value-line-border"; - imageLine.appendChild(tensorToImage(JSON.parse(initializer.toString()), 2, 3, this._host.document)); - this._element.appendChild(imageLine); - } catch (err) { - // do nothing - } + this._element.appendChild(new sidebar.VisualTensorView(this._host, JSON.parse(initializer.toString()))._element); } } catch (err) { contentLine.innerHTML = err.toString(); From e46680256dfab9480bebf940ba161d5b75f09e6c Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Mon, 6 Feb 2023 18:33:15 +0300 Subject: [PATCH 03/11] Add values selector --- media/CircleGraph/view-sidebar.css | 2 + media/CircleGraph/view-sidebar.js | 110 +++++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/media/CircleGraph/view-sidebar.css b/media/CircleGraph/view-sidebar.css index 643413ef..b55ab067 100644 --- a/media/CircleGraph/view-sidebar.css +++ b/media/CircleGraph/view-sidebar.css @@ -70,6 +70,8 @@ https://github.com/lutzroeder/netron/blob/ae449ff55642636e6a1eef092eda34ffcba1c6 .sidebar-view-item-value-line-content { white-space: pre; word-wrap: normal; overflow: auto; display: block; } .sidebar-view-item-value-expander { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; float: right; color: #aaa; cursor: pointer; user-select: none; -webkit-user-select: none; -moz-user-select: none; padding: 4px 6px 4px 6px; } .sidebar-view-item-value-expander:hover { color: #000; } +.sidebar-view-item-value-number { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, Courier, monospace; text-align: center; float: left; font-size: 11px !important; width: 14px !important; height: 11px !important; } +.sidebar-view-item-value-number::-webkit-outer-spin-button, .sidebar-view-item-value-number::-webkit-inner-spin-button { -webkit-appearance: none; margin: 0; } .sidebar-view-item-select { font-family: inherit; font-size: 12px; background-color: #fcfcfc; border: #fcfcfc; color: #333; diff --git a/media/CircleGraph/view-sidebar.js b/media/CircleGraph/view-sidebar.js index 8b334088..e1575a63 100644 --- a/media/CircleGraph/view-sidebar.js +++ b/media/CircleGraph/view-sidebar.js @@ -643,14 +643,13 @@ function getTensorValue(tensor, index) { return value; } -function normalizeTensor(tensor, axis1, axis2, values = {}) { +function normalizeTensor(tensor, axis1, axis2, values) { let shape = getTensorShape(tensor); let height = shape[axis1]; let width = shape[axis2]; let imageData = []; - let index = Array(shape.length); - index.fill(0); + let index = values.slice(); // find min and max values in the tensor let minValue = Infinity; @@ -679,7 +678,7 @@ function normalizeTensor(tensor, axis1, axis2, values = {}) { return imageData; } -function tensorToImage(tensor, axis1, axis2, document, values = {}) { +function tensorToImage(tensor, axis1, axis2, values, document) { let scale = 12; let imageData = normalizeTensor(tensor, axis1, axis2, values); let height = imageData.length; @@ -722,18 +721,35 @@ sidebar.VisualTensorView = class { this._tensor = tensor; this._element = this._host.document.createElement("div"); - this._element.class = "sidebar-view-item-value-line-border"; - this._tensorShape = getTensorShape(this._tensor); if (this._tensorShape.length < 2) { return this; } + this._element.className = "sidebar-view-item-value-line-border"; + this._axes = [this._tensorShape.length - 2, this._tensorShape.length - 1]; - // add axes selection checkboxes + this._values = Array(this._tensorShape.length); + this._values.fill(0); if (this._tensorShape.length > 2) { this._checkboxes = []; + this._valueTexts = []; + this._infoTexts = []; for (let i = 0; i < this._tensorShape.length; ++i) { + let infoText = this._host.document.createElement("div"); + infoText.setAttribute('style', 'float: left;'); + infoText.setText = () => { + infoText.innerHTML = "/ " + (this._tensorShape[i] - 1); + if (this._axes[0] === i) { + infoText.innerHTML += " (x)"; + } + else if (this._axes[1] === i) { + infoText.innerHTML += " (y)"; + } + }; + this._infoTexts.push(infoText); + let checkbox = this._host.document.createElement("input"); + checkbox.setAttribute('style', 'float: left;'); checkbox.type = "checkbox"; checkbox.checked = this._axes.includes(i); checkbox.addEventListener("change", () => { @@ -747,15 +763,85 @@ sidebar.VisualTensorView = class { this._axes.splice(this._axes.indexOf(i), 1); } this.updateImage(); + this.updateUI(); }); this._checkboxes.push(checkbox); + + let valueText = this._host.document.createElement("input"); + valueText.className = "sidebar-view-item-value-number"; + valueText.type = "number"; + valueText.min = 0; + valueText.max = this._tensorShape[i] - 1; + valueText.value = 0; + valueText.setValue = (value) => { + if (valueText.disabled) { + return; + } + valueText.value = value; + if (valueText.value === "") { + valueText.value = 0; + } + if (parseInt(valueText.value) < parseInt(valueText.min)) { + valueText.value = valueText.min; + } + if (parseInt(valueText.value) > parseInt(valueText.max)) { + valueText.value = valueText.max; + } + this._values[i] = parseInt(valueText.value); + this.updateImage(); + }; + valueText.decrease = () => { + if (valueText.value === valueText.min) { + valueText.setValue(valueText.max); + } else { + valueText.setValue(parseInt(valueText.value) - 1); + } + }; + valueText.increase = () => { + if (valueText.value === valueText.max) { + valueText.setValue(valueText.min); + } else { + valueText.setValue(parseInt(valueText.value) + 1); + } + }; + valueText.addEventListener("change", () => { + valueText.setValue(valueText.value); + }); + this._valueTexts.push(valueText); + + let leftButton = this._host.document.createElement("div"); + leftButton.className = "sidebar-view-item-value-expander"; + leftButton.setAttribute('style', 'float: left; padding: 1px 4px 0px 4px;'); + leftButton.innerHTML = "ᐊ"; + leftButton.addEventListener("click", valueText.decrease); + + let rightButton = this._host.document.createElement("div"); + rightButton.className = "sidebar-view-item-value-expander"; + rightButton.setAttribute('style', 'float: left; padding: 1px 4px 0px 4px;'); + rightButton.innerHTML = "ᐅ"; + rightButton.addEventListener("click", valueText.increase); + this._element.appendChild(checkbox); + this._element.appendChild(leftButton); + this._element.appendChild(valueText); + this._element.appendChild(rightButton); + this._element.appendChild(infoText); + + this._element.appendChild(this._host.document.createElement("br")); + this._element.appendChild(this._host.document.createElement("br")); } } - this._element.appendChild(this._host.document.createElement("br")); this._image = null; this.updateImage(); + this.updateUI(); + } + + updateUI() { + for (let i = 0; i < this._valueTexts.length; ++i) { + this._infoTexts[i].setText(); + this._valueTexts[i].disabled = this._axes.includes(i); + } } updateImage() { @@ -767,7 +853,7 @@ sidebar.VisualTensorView = class { return; } try { - this._image = tensorToImage(this._tensor, this._axes[0], this._axes[1], this._host.document); + this._image = tensorToImage(this._tensor, this._axes[1], this._axes[0], this._values, this._host.document); this._element.appendChild(this._image); } catch (err) { // do nothing @@ -912,6 +998,9 @@ sidebar.ArgumentView = class { const valueLine = this._host.document.createElement("div"); try { const state = initializer.state; + if (!state) { + this._element.appendChild(new sidebar.VisualTensorView(this._host, JSON.parse(initializer.toString()))._element); + } if ( state === null && this._host.save && @@ -932,9 +1021,6 @@ sidebar.ArgumentView = class { valueLine.className = "sidebar-view-item-value-line-border"; contentLine.innerHTML = state || initializer.toString(); - if (!state) { - this._element.appendChild(new sidebar.VisualTensorView(this._host, JSON.parse(initializer.toString()))._element); - } } catch (err) { contentLine.innerHTML = err.toString(); this._raise("error", err); From b35c070f7cafc7a0d74343da504c9e090a5a9fd3 Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Mon, 6 Feb 2023 19:36:27 +0300 Subject: [PATCH 04/11] Add value tooltip --- media/CircleGraph/view-sidebar.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/media/CircleGraph/view-sidebar.js b/media/CircleGraph/view-sidebar.js index e1575a63..dd6dd382 100644 --- a/media/CircleGraph/view-sidebar.js +++ b/media/CircleGraph/view-sidebar.js @@ -679,16 +679,25 @@ function normalizeTensor(tensor, axis1, axis2, values) { } function tensorToImage(tensor, axis1, axis2, values, document) { - let scale = 12; + const scale = 12; let imageData = normalizeTensor(tensor, axis1, axis2, values); let height = imageData.length; let width = imageData[0].length; let canvas = document.createElement('canvas'); + canvas.imageData = imageData; canvas.width = width * scale; canvas.height = height * scale; let ctx = canvas.getContext('2d'); ctx.imageSmoothingEnabled = false; let imageDataArray = new Uint8ClampedArray(height * width * 4); + canvas.getValue = (clientX, clientY) => { + const rect = canvas.getBoundingClientRect(); + const x = Math.floor(Math.min(Math.max(clientX - rect.left, 0), canvas.width - 0.1) / scale); + const y = Math.floor(Math.min(Math.max(clientY - rect.top, 0), canvas.height - 0.1) / scale); + // for now the function returns normalized values + // TODO: return original values + return canvas.imageData[y][x]; + }; for (let i = 0; i < height; i++) { for (let j = 0; j < width; j++) { let value = imageData[i][j]; @@ -832,7 +841,7 @@ sidebar.VisualTensorView = class { } } - this._image = null; + this._imageContainer = null; this.updateImage(); this.updateUI(); } @@ -845,16 +854,21 @@ sidebar.VisualTensorView = class { } updateImage() { - if (this._image) { - this._element.removeChild(this._image); + if (this._imageContainer) { + this._element.removeChild(this._imageContainer); } - this._image = null; + this._imageContainer = null; if (this._axes.length < 2) { return; } try { + this._imageContainer = this._host.document.createElement("div"); this._image = tensorToImage(this._tensor, this._axes[1], this._axes[0], this._values, this._host.document); - this._element.appendChild(this._image); + this._image.addEventListener("mousemove", (e) => { + this._imageContainer.title = this._image.getValue(e.clientX, e.clientY); + }, false); + this._imageContainer.appendChild(this._image); + this._element.appendChild(this._imageContainer); } catch (err) { // do nothing } From 71fa4a86d5516f7652a468b6d44a0866d90d6ad1 Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Wed, 8 Feb 2023 12:02:36 +0300 Subject: [PATCH 05/11] Show original values in tooltip --- media/CircleGraph/view-sidebar.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/media/CircleGraph/view-sidebar.js b/media/CircleGraph/view-sidebar.js index dd6dd382..69ea8cb0 100644 --- a/media/CircleGraph/view-sidebar.js +++ b/media/CircleGraph/view-sidebar.js @@ -684,7 +684,6 @@ function tensorToImage(tensor, axis1, axis2, values, document) { let height = imageData.length; let width = imageData[0].length; let canvas = document.createElement('canvas'); - canvas.imageData = imageData; canvas.width = width * scale; canvas.height = height * scale; let ctx = canvas.getContext('2d'); @@ -694,9 +693,10 @@ function tensorToImage(tensor, axis1, axis2, values, document) { const rect = canvas.getBoundingClientRect(); const x = Math.floor(Math.min(Math.max(clientX - rect.left, 0), canvas.width - 0.1) / scale); const y = Math.floor(Math.min(Math.max(clientY - rect.top, 0), canvas.height - 0.1) / scale); - // for now the function returns normalized values - // TODO: return original values - return canvas.imageData[y][x]; + let index = values.slice(); + index[axis1] = y; + index[axis2] = x; + return getTensorValue(tensor, index); }; for (let i = 0; i < height; i++) { for (let j = 0; j < width; j++) { From be48d1df05e8bf8e0eada95d9a8dd806c4a95200 Mon Sep 17 00:00:00 2001 From: Stanislav Ponkrashov Date: Wed, 8 Feb 2023 15:15:11 +0300 Subject: [PATCH 06/11] Enable VisualTensorView only for CircleGraph --- media/CircleGraph/index.html | 2 ++ media/CircleGraph/view-sidebar.js | 2 +- src/CircleGraph/CircleGraphCtrl.ts | 8 ++++++++ src/CircleGraph/CircleViewer.ts | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/media/CircleGraph/index.html b/media/CircleGraph/index.html index 9072ea67..146195b7 100644 --- a/media/CircleGraph/index.html +++ b/media/CircleGraph/index.html @@ -72,6 +72,8 @@