FuzzCodeErrMinRunnerTests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4df78c78-1e5c-49c8-98b7-08080a478e4d",
   "metadata": {},
   "source": [
    "(C) 2023-24 Manfred Schlaegl <manfred.schlaegl@jku.at>, Institute for Complex Systems, JKU Linz\n",
    "\n",
    "SPDX-License-Identifier: BSD 3-clause \"New\" or \"Revised\" License"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6056b1db-968d-40db-ab73-50c700753b7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import rvvts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e903e369-97ae-47c8-9ace-117fd848bcc7",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "865ba3f1-3064-4ae9-b81d-3d9cf9bb4260",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run config_internal.ipynb\n",
    "%run config_host.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "385240cb-a3f2-4b48-b7b1-6c98601a1e66",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAIN SETTINGS\n",
    "instance = 0\n",
    "\n",
    "dut = \"VP\"\n",
    "#dut = \"QEMU\"\n",
    "\n",
    "xlen = 32\n",
    "#xlen = 64\n",
    "    \n",
    "#vector_vlen = 128\n",
    "vector_vlen = 512\n",
    "\n",
    "\n",
    "# build config\n",
    "config = dict()\n",
    "config.update(config_internal.copy())\n",
    "config.update(config_host.copy())\n",
    "config.update(dict(\n",
    "    dir = os.getcwd() + \"/run/FuzzCodeErrMinRunner_\" + str(instance) + \"_\" + dut,\n",
    "\n",
    "    ArchiveRunner_dut = rvvts.FuzzCodeErrMinRunner,\n",
    "    \n",
    "    debug_port = 8000 + instance,\n",
    "\n",
    "    CompareRunner_dut = rvvts.DuTGDBRunner,\n",
    "\n",
    "    # Optional: Coverage\n",
    "    #RefCovRunner_coverage = rvvts.RISCVOVPSIMCoverageRunner,\n",
    "    #RISCVOVPSIMCover_metric = \"basic\", # basic, extended or mnemonic\n",
    "    #RISCVOVPSIMCover_sum_enable = True,\n",
    "    \n",
    "    xlen = xlen,\n",
    "    vector_vlen = vector_vlen,\n",
    "))\n",
    "\n",
    "if dut == \"VP\":\n",
    "    config[\"DuTGDBRunner_dut\"] = rvvts.VPRunner\n",
    "elif dut == \"QEMU\":\n",
    "    config[\"DuTGDBRunner_dut\"] = rvvts.QEMURunner\n",
    "else:\n",
    "    raise Exception(\"invalid dut\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5792edff-8ce2-434b-8dae-42afea996163",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ede87c0-4398-4fd7-b32c-a50b062cb632",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "runner = rvvts.ArchiveRunner(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8399ad67-929f-43f6-9179-f581a2225c8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9e343def-1021-4039-b2a3-330675c970dd",
   "metadata": {},
   "source": [
    "## First Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c53da23-7bd8-41de-9d89-91b5589f99f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#res = runner.run(min_fragments = 10, max_fragments = 100, blocking = True, timeout = 2.0)\n",
    "#res = runner.run(min_fragments = 10000, max_fragments = 10000, blocking = True, timeout = 2.0)\n",
    "res = runner.run(min_fragments = 5000, max_fragments = 5000, blocking = True, timeout = 2.0)\n",
    "\n",
    "print(res[0])\n",
    "print(res[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6b28080-e62a-4c4c-88ad-de028bc318db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print code (in case of error)\n",
    "#print(len(runner.ArchiveRunner_dut.res_code_block.as_code()))\n",
    "#print(runner.ArchiveRunner_dut.res_code_block.as_code())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05aa69ab-3d90-4493-af10-15f0bfc0c1f9",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "## Manual Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "003e816b-e4d5-429d-a3bb-2f333f1db6d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manually generated test-code, or generated code (e.g. run/FuzzCodeErrMinRunner_0_VP/ArchiveRunner_xxx/ERROR_xxxxxx_iteration_xxxxxxxxxx/CodeErrMinRunner_0/CodeCompareRunner_0/BuildRunner_0/code.S)\n",
    "code = \"\"\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48b6e6ea-947c-48ad-b56e-f0b23ef341e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#ret = runner.ArchiveRunner_dut.codecomparerunner.run(timeout = 2.0, blocking = True, code = runner.ArchiveRunner_dut.code.as_code())\n",
    "ret = runner.ArchiveRunner_dut.codeerrminrunner.codecomparerunner.run(timeout = 2.0, blocking = True, code = code)\n",
    "#print(code)\n",
    "print(ret[0])\n",
    "print(ret[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7622229-4c98-45b8-903e-22855c88c7a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e7eb94d8-17a8-4ea8-aa7d-31fafca7e834",
   "metadata": {},
   "source": [
    "## Automated Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b2737eb-df1e-429d-a69e-4774b8360801",
   "metadata": {},
   "outputs": [],
   "source": [
    "def FuzzCodeErrMin_stats(runner):\n",
    "    cemr = runner.ArchiveRunner_dut.codeerrminrunner\n",
    "    covr = cemr.codecomparerunner.compare_runner.CompareRunner_refcov.RefCovRunner_cov\n",
    "\n",
    "    out = \", SUM: [ tests: \" + str(cemr.tests) +\\\n",
    "            \", completes: \" + str(cemr.completes) +\\\n",
    "            \", ignores: \" + str(cemr.ignores) +\\\n",
    "            \", timeouts: \" + str(cemr.timeouts) +\\\n",
    "            \", unknown_faults: \" + str(cemr.unknown_faults) +\\\n",
    "            \", errors: \" + str(cemr.errors) +\\\n",
    "            \", reductions: \" + str(cemr.reductions) +\\\n",
    "            \", minimizations: \" + str(cemr.minimizations) +\\\n",
    "            \", instr_errors: \" + str(cemr.instr_errors)\n",
    "\n",
    "    if covr:\n",
    "        coverage_type = covr.coverage[\"sum\"][\"coverage\"][\"type\"]\n",
    "        coverage = covr.coverage[\"sum\"][\"coverage\"][\"percent\"]\n",
    "        out += \", coverage (\" + str(coverage_type) + \"): \" + str(coverage) + \"%\"\n",
    "\n",
    "    out += \" ] \"\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05746d69-56ca-4892-b9ba-7c810ad22a5a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "rvvts.runner_bench(\n",
    "    runner,\n",
    "    custom_stat_f = FuzzCodeErrMin_stats,\n",
    "    iter = 10,\n",
    "    min_fragments = 5000,\n",
    "    max_fragments = 5000,\n",
    "    timeout = 3.0,\n",
    "    #stop_on_error = True,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}