From 0185461f9ccebb9516b52afa4948212619a2f9d4 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 29 Jun 2024 01:51:26 +0300 Subject: [PATCH 01/18] add `sqlite3.ll` to avoid accidental commits --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6d9944c8..945affdf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ seed_embeddings/preprocessed/* html latex +sqlite3.ll \ No newline at end of file From 0a7cf9830f6b6375362e3636654d4bcbafad8c0a Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 29 Jun 2024 02:04:57 +0300 Subject: [PATCH 02/18] add sqlite llvm ir file generation --- src/test-suite/CMakeLists.txt | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index 297a1db9..dacb40b8 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -8,3 +8,69 @@ file(COPY index-llvm17.files DESTINATION ./) configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) file(COPY test-lit.py DESTINATION ./) file(COPY test-ir2vec.lit DESTINATION ./) + +# SQLite IR file is added along side other PE-benchmarks for this version +set(SQLITE_LLVM_IR + ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll +) + +if(NOT EXISTS ${SQLITE_LLVM_IR}) + # The downloaded Zip file and source files directory are located at the + # build directory since its already gitignore anyways. + set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip) + set(SQLITE_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000) + + # Some of the recommended compile flags. https://sqlite.org/amalgamation.html + # with the following flags the resulting LLVM IR file is about 500k lines + set(SQLITE_COMPILE_FLAGS + -DSQLITE_ENABLE_FTS3 + -DSQLITE_ENABLE_FTS5 + -DSQLITE_ENABLE_RTREE + -DSQLITE_ENABLE_DBSTAT_VTAB + -DSQLITE_ENABLE_RBU + -DSQLITE_ENABLE_SESSION + ) + + if(NOT EXISTS ${SQLITE_ZIP}) + # SQLite Amalgamation download source: https://sqlite.org/download.html + message(STATUS "Downloading SQLite Amalgamation source zip file..") + file( + DOWNLOAD https://sqlite.org/2024/sqlite-amalgamation-3460000.zip + ${SQLITE_ZIP} + EXPECTED_HASH SHA3_256=1221eed70de626871912bfca144c00411f0c30d3c2b7935cff3963b63370ef7c + SHOW_PROGRESS + ) + endif() + + if(NOT EXISTS ${SQLITE_SOURCE}) + message(STATUS + "Unzipping SQLite source files at: ${SQLITE_SOURCE}" + ) + execute_process( + COMMAND unzip ${SQLITE_ZIP} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE unzip_result + ) + + if(NOT unzip_result EQUAL 0) + message(FATAL_ERROR "Failed to extract SQLite zip file") + endif() + endif() + + find_program(CLANG_BIN NAMES clang-17 clang) + if(NOT CLANG_BIN) + message(FATAL_ERROR "Clang not found. Please install it.") + endif() + + execute_process( + COMMAND ${CLANG_BIN} -S -emit-llvm ${SQLITE_COMPILE_FLAGS} + -Xclang -disable-O0-optnone + -o ${SQLITE_LLVM_IR} + ${SQLITE_SOURCE}/sqlite3.c + RESULT_VARIABLE sqlite_ir_result + ) + + if(sqlite_ir_result EQUAL 0) + message(STATUS "SQLite benchmark was generated at: ${SQLITE_LLVM_IR}") + endif() +endif() \ No newline at end of file From cc6f3f5473bae0211a5ba92125895db18916dece Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 29 Jun 2024 02:30:18 +0300 Subject: [PATCH 03/18] fix pre-commit --- .gitignore | 2 +- src/test-suite/CMakeLists.txt | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 945affdf..e1cdfc51 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ seed_embeddings/preprocessed/* html latex -sqlite3.ll \ No newline at end of file +sqlite3.ll diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index dacb40b8..513a0b24 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -10,25 +10,25 @@ file(COPY test-lit.py DESTINATION ./) file(COPY test-ir2vec.lit DESTINATION ./) # SQLite IR file is added along side other PE-benchmarks for this version -set(SQLITE_LLVM_IR +set(SQLITE_LLVM_IR ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll ) if(NOT EXISTS ${SQLITE_LLVM_IR}) - # The downloaded Zip file and source files directory are located at the + # The downloaded Zip file and source files directory are located at the # build directory since its already gitignore anyways. set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip) set(SQLITE_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000) # Some of the recommended compile flags. https://sqlite.org/amalgamation.html # with the following flags the resulting LLVM IR file is about 500k lines - set(SQLITE_COMPILE_FLAGS + set(SQLITE_COMPILE_FLAGS -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_RBU - -DSQLITE_ENABLE_SESSION + -DSQLITE_ENABLE_SESSION ) if(NOT EXISTS ${SQLITE_ZIP}) @@ -36,14 +36,14 @@ if(NOT EXISTS ${SQLITE_LLVM_IR}) message(STATUS "Downloading SQLite Amalgamation source zip file..") file( DOWNLOAD https://sqlite.org/2024/sqlite-amalgamation-3460000.zip - ${SQLITE_ZIP} - EXPECTED_HASH SHA3_256=1221eed70de626871912bfca144c00411f0c30d3c2b7935cff3963b63370ef7c + ${SQLITE_ZIP} + EXPECTED_HASH SHA3_256=1221eed70de626871912bfca144c00411f0c30d3c2b7935cff3963b63370ef7c SHOW_PROGRESS ) endif() if(NOT EXISTS ${SQLITE_SOURCE}) - message(STATUS + message(STATUS "Unzipping SQLite source files at: ${SQLITE_SOURCE}" ) execute_process( @@ -63,9 +63,9 @@ if(NOT EXISTS ${SQLITE_LLVM_IR}) endif() execute_process( - COMMAND ${CLANG_BIN} -S -emit-llvm ${SQLITE_COMPILE_FLAGS} - -Xclang -disable-O0-optnone - -o ${SQLITE_LLVM_IR} + COMMAND ${CLANG_BIN} -S -emit-llvm ${SQLITE_COMPILE_FLAGS} + -Xclang -disable-O0-optnone + -o ${SQLITE_LLVM_IR} ${SQLITE_SOURCE}/sqlite3.c RESULT_VARIABLE sqlite_ir_result ) @@ -73,4 +73,4 @@ if(NOT EXISTS ${SQLITE_LLVM_IR}) if(sqlite_ir_result EQUAL 0) message(STATUS "SQLite benchmark was generated at: ${SQLITE_LLVM_IR}") endif() -endif() \ No newline at end of file +endif() From 7fa189b2e8f4c0580b2b9ac9844990e39f4023df Mon Sep 17 00:00:00 2001 From: m-atalla Date: Fri, 5 Jul 2024 23:21:29 +0300 Subject: [PATCH 04/18] add `ENABLE_SQLITE` option --- src/test-suite/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index 513a0b24..2c5ea1ab 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -9,12 +9,14 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) file(COPY test-lit.py DESTINATION ./) file(COPY test-ir2vec.lit DESTINATION ./) +option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON) + # SQLite IR file is added along side other PE-benchmarks for this version set(SQLITE_LLVM_IR ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll ) -if(NOT EXISTS ${SQLITE_LLVM_IR}) +if(NOT EXISTS ${SQLITE_LLVM_IR} AND SQLITE_ENABLED) # The downloaded Zip file and source files directory are located at the # build directory since its already gitignore anyways. set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip) From ce3d9604e9657308f4a378dd7502d3390fd871f3 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 6 Jul 2024 00:20:38 +0300 Subject: [PATCH 05/18] add sqlite test oracles --- src/test-suite/oracle/FA_llvm17_p/sqlite3.txt | 1 + src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 src/test-suite/oracle/FA_llvm17_p/sqlite3.txt create mode 100644 src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt diff --git a/src/test-suite/oracle/FA_llvm17_p/sqlite3.txt b/src/test-suite/oracle/FA_llvm17_p/sqlite3.txt new file mode 100644 index 00000000..b3291227 --- /dev/null +++ b/src/test-suite/oracle/FA_llvm17_p/sqlite3.txt @@ -0,0 +1 @@ +39389.228010 -5376.674414 -29021.478687 57311.999476 -2830.058969 -4903.917802 12531.761407 11238.552214 5837.613706 -75945.222073 -29524.528980 67012.326909 12066.566763 -3894.166052 -5155.274300 24880.990511 16379.301698 18119.941570 -78168.641123 6989.300529 12382.527812 -4140.765868 -22445.730997 -11017.107544 -2946.210374 -24964.022127 31784.060329 24619.920816 3814.250199 53991.318974 -15070.562586 30707.538848 43953.849707 -2381.574458 22950.984229 41734.144641 -10816.221965 -4810.135903 -22139.721114 -18213.911622 65218.353935 28923.175315 -4972.659720 25150.057632 -1473.603306 60679.434496 14821.739239 48281.290493 15413.638264 -21271.516911 17702.110667 29020.863980 -31464.468252 -12264.696262 -53225.776560 4551.504887 20417.821305 12091.340519 -24138.628052 28349.748690 -17150.800330 12706.377159 24665.181318 16145.634347 65299.241720 21431.810517 40498.453896 51106.281433 54662.926106 -40088.949771 13304.092762 7654.656533 14419.047690 1069.853817 48025.749385 -22396.022974 -42850.026510 -6809.055547 -521.476994 -64375.753597 -7862.747134 -5017.845790 81997.642405 -46507.572785 -7365.529276 1302.849919 25304.077449 58076.845632 -34610.699881 4141.103685 -51849.889573 -3722.191701 -9419.574215 79247.276286 9473.697538 19713.967952 -12465.907796 19244.744645 -3435.023306 13989.461256 6536.506768 -56492.493102 -1688.045743 -73137.700853 -29253.134499 -40407.956653 -933.743462 14279.813178 -57154.836462 557.310850 4767.098763 53947.134808 -84054.927478 -5960.262662 15618.135863 6821.010998 52393.624293 12726.230237 -20777.351490 9294.755970 1390.254836 17729.270299 23544.548798 -26589.157997 11527.892553 -2477.706692 2599.986765 10135.742292 17323.593387 18006.984562 -84040.870641 -13220.164514 23623.495802 -7462.858326 7428.499912 29600.097469 25956.154793 -22568.528168 22848.602044 -41683.667110 46822.194460 -63180.261022 -16181.937610 -23577.152061 -38918.014782 -27130.354619 40871.462790 45417.167782 45708.189488 -8005.036744 -9525.828909 -574.773616 -72716.434262 -90806.812223 10465.695405 -1704.417641 -3791.325645 11437.562356 -9734.447109 10740.288521 -43880.468152 14900.821761 34527.372778 -29568.994163 -26237.248706 -3183.854486 44694.863175 10560.089449 -18749.784008 -15393.422626 -7807.926404 -3875.797153 23007.442336 54734.520268 -18045.888336 -91677.309373 -81314.052401 -25529.987834 49294.973712 -25441.928778 -4832.642230 -21895.379659 -25103.559014 -31507.116894 -2475.733488 -39250.694689 12937.243692 -6512.779912 -17903.734550 -64470.347896 22094.732030 -9237.200299 -14267.520417 10192.716821 -47076.708383 10964.664559 30742.866710 4235.252363 1191.728237 10027.339188 4415.971911 -38850.183036 -29519.167851 20492.481769 -23659.175199 52721.378257 42300.583697 2340.791768 -29988.064784 -19580.261322 -49547.471529 33755.555148 -48905.776848 35422.181670 8024.147986 -27027.556318 33921.390322 -44944.517675 -7937.652834 -29487.218742 26365.878881 6197.248727 -71044.945394 70024.038121 51335.191099 -42336.803306 -3397.896498 -9550.309290 23323.897056 11667.371271 39492.749377 -43755.789585 -36460.205077 14610.025897 151.649603 -1885.533970 -2971.605785 14692.409094 -12728.623378 -2336.804940 -48256.526733 -15118.845499 47250.475424 26403.952496 -6263.492780 1761.811526 13312.365912 -25423.677786 12047.566917 49352.438572 30559.417573 12879.122901 -45558.117241 -102363.066526 21858.284461 -27296.611234 -9673.486010 6833.930802 -27911.443354 -18580.310923 16977.709576 -28889.467960 48066.249561 -17385.422090 -33583.855529 46193.509969 -69949.758441 -54972.692493 -46011.694699 -13363.262082 11685.615563 -4471.066728 22318.769605 -21219.305738 9400.389388 -7905.007664 19022.545764 -19689.155351 -29574.226282 12494.780265 28997.563923 45806.675927 7127.876997 -14628.763346 57523.118124 9747.445151 -12639.944425 -14974.659152 -5337.028753 -25489.188876 8811.413006 -66892.993020 -10358.495763 4699.510652 33516.701552 -40010.545447 22338.301792 70.999160 17700.864559 10359.928443 diff --git a/src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt b/src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt new file mode 100644 index 00000000..7e5ab73b --- /dev/null +++ b/src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt @@ -0,0 +1 @@ +16914.214172 1840.963612 -8056.768579 21579.305453 -906.262205 -14727.750711 4137.897728 12411.245494 -6586.577729 -37492.964734 -23503.579107 24019.665282 3178.140591 -4723.500753 5407.842244 7912.921167 -3458.007608 3013.286848 -43144.929418 15115.546246 13531.053632 4908.258806 -5820.302126 -7871.660645 3889.551273 -10608.906240 1448.299046 10127.834420 1014.800933 22875.036123 -11217.396606 7132.652507 18359.679961 -541.432431 9754.327113 19447.189792 -5458.297947 1001.803515 1260.266054 429.515246 30198.629348 18262.393044 -3520.028532 10885.129589 9038.867032 23509.461660 -2190.924487 20668.772705 -4464.237518 -15003.251139 10138.776080 7383.103845 -17749.750384 -2971.023222 -14480.273423 9624.178060 19172.138324 8872.303230 -11147.777716 11520.362299 -11789.698651 -2464.035909 7416.324752 381.487262 27537.998172 14634.787225 15946.290784 14708.178722 30142.986762 -11951.156505 -2847.544498 13036.279684 11946.873288 -2281.002892 14178.835480 -9445.563955 -21959.072204 -11427.626110 14.950048 -38164.969681 -15326.407916 -604.005724 40969.720254 -22921.026771 1021.245459 6482.292367 8335.949495 22709.037213 -15671.772052 -634.916992 -16388.880672 -88.372465 -10411.435811 38141.528995 1444.237535 7092.407774 -3015.810814 9673.987822 10480.803898 17367.615576 -5099.230566 -15713.135321 -16277.581293 -37081.163429 -8470.125739 -8772.651484 3261.668365 7589.841179 -31083.117699 -6787.744450 7673.834177 27727.294920 -34264.453388 -1024.019126 18412.358452 9384.182441 16855.866440 7579.575966 -20625.224866 4570.472021 1587.196202 1714.091642 13996.430750 -8729.059297 -1445.531903 -5345.869665 6737.841282 747.071812 5712.435997 1316.571824 -37393.893115 -11137.285625 3457.923217 -7597.644020 934.439596 4310.015852 8189.372290 -13761.423625 12699.496546 -14782.421306 18183.527744 -33691.703352 -517.262789 -11239.399834 -19974.966452 -3504.060581 16885.975472 25761.155805 14053.165475 -6681.317758 -6153.418469 -7325.079859 -30356.658229 -34656.871696 6283.765911 -1758.163001 8988.464773 -1522.551626 -1673.995728 5009.828211 -9656.368676 1426.665633 9335.348034 -11649.556808 -1566.318860 -6295.734684 31544.197891 661.649733 -4830.674434 -14478.650575 -8427.805505 -545.973874 2502.690463 15002.196104 -6300.676337 -33845.085246 -34194.679896 -18256.095681 19120.451916 -4796.204741 -1357.198324 -13606.642437 -3021.046722 -18445.044067 3332.247063 -17445.313795 6821.379552 -1561.129509 -13353.268278 -23115.474322 472.936692 -1898.996769 -9484.504976 -5017.053747 -15333.591479 -5749.949470 18458.335412 4047.862621 8084.542286 3028.814311 1129.188741 -21428.210033 -19353.038396 5786.410664 -7532.951615 21430.806942 19577.147353 459.479687 -18445.971763 -4433.705096 -27620.540913 19828.835893 -17033.978568 4474.063320 -6138.081727 -9289.859715 29276.565507 -13631.410108 -7187.016573 -18083.697502 8781.137185 3854.077762 -34757.682776 32445.021522 20843.803376 -11541.272980 -4382.951260 -1828.406638 2398.068724 18359.047156 21539.259133 -15484.985073 -13341.607581 3250.685126 -6791.923469 -1860.182208 -395.840937 16642.290996 -16803.371117 11532.507834 -15286.484050 -9243.681272 12537.733834 16148.825960 493.864148 8958.987453 -833.191497 -2730.725876 8574.123858 17163.311834 13874.951473 -6250.890749 -19190.295017 -44012.399335 13558.447731 -8566.129714 -6785.466744 -403.105960 -9429.032390 -19264.410796 3993.546581 -5802.231716 24354.598056 802.353023 -11928.318532 16617.764262 -27415.844372 -24580.951602 -13247.186538 2753.917039 -291.838695 -5530.368520 7072.636033 -7053.490037 6451.836519 9084.519734 856.179694 1720.038980 -17071.204665 -10241.571031 -236.670560 18695.744940 8736.999602 -4602.100388 24403.244235 5754.482284 -10343.410231 -6480.548159 858.463332 31.853096 -2285.052285 -20497.891846 -9698.522648 -9221.781745 8000.340915 -9367.694417 14972.296732 1337.978384 1709.723559 9941.290745 From e8c138febdd2594c6f34dcffe744f19f3425374e Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 6 Jul 2024 00:21:09 +0300 Subject: [PATCH 06/18] add sqlite tests --- src/test-suite/sanity_check.sh.cmake | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 00b59e16..95a4d418 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -43,6 +43,14 @@ perform_vector_comparison() { ORIG_FILE=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/ir2vec.txt VIR_FILE=ir2vec_${FILE_PREFIX}.txt + + # SQLite specific variables.. Only declared + if [[ "@ENABLE_SQLITE@" == "ON" ]]; then + SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt + SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll + SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt + fi + # if file prefix is p or f, run the first while loop, else, run the second while loop if [[ "$FILE_PREFIX" == "p" || "$FILE_PREFIX" == "f" ]]; then @@ -50,6 +58,11 @@ perform_vector_comparison() { ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${VIR_FILE} ${d} &> /dev/null done < index-${SEED_VERSION}.files wait + + # SQLITE is currently only tested against the program (p) level + if [[ "@ENABLE_SQLITE@" == "ON" && "$FILE_PREFIX" == "p" ]]; then + ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null + fi else while IFS= read -r d_on do @@ -78,6 +91,25 @@ perform_vector_comparison() { echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}" exit 1 fi + + # SQLite tests only if its enabled + if [[ "@ENABLE_SQLITE@" == "ON" ]]; then + if [[ ! -e "$SQLITE_VIR" ]]; then + echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" + exit 1 + fi + mv ${SQLITE_VIR} ${TEMP}/ + + d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${SQLITE_ORIG}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${SQLITE_VIR})) + + if [ "$d" == "" ]; then + echo -e "${GREEN}${BOLD}[Test Passed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" + else + echo -e "$(tput bold)${RED}[Test Failed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" + exit 1 + fi + fi + else if ls *${VIR_FILE} 1> /dev/null 2>&1 then From fc756892a4efdd6676dbc7013e36f0939da68e8a Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 6 Jul 2024 00:22:21 +0300 Subject: [PATCH 07/18] update --- src/test-suite/sanity_check.sh.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 95a4d418..237352bf 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -44,7 +44,7 @@ perform_vector_comparison() { VIR_FILE=ir2vec_${FILE_PREFIX}.txt - # SQLite specific variables.. Only declared + # SQLite specific variables. if [[ "@ENABLE_SQLITE@" == "ON" ]]; then SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll From fb600bf11a786495a8229825be0a1f423ef1c03f Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 6 Jul 2024 00:30:44 +0300 Subject: [PATCH 08/18] update --- src/test-suite/sanity_check.sh.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 237352bf..66450d02 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -92,7 +92,7 @@ perform_vector_comparison() { exit 1 fi - # SQLite tests only if its enabled + # SQLite tests only if its enabled if [[ "@ENABLE_SQLITE@" == "ON" ]]; then if [[ ! -e "$SQLITE_VIR" ]]; then echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" From a620100681e46721f6335fdce7712ea70088988d Mon Sep 17 00:00:00 2001 From: m-atalla Date: Sat, 6 Jul 2024 00:37:42 +0300 Subject: [PATCH 09/18] update --- src/test-suite/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index 2c5ea1ab..9204ec47 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -16,7 +16,7 @@ set(SQLITE_LLVM_IR ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll ) -if(NOT EXISTS ${SQLITE_LLVM_IR} AND SQLITE_ENABLED) +if(NOT EXISTS ${SQLITE_LLVM_IR} AND ENABLE_SQLITE) # The downloaded Zip file and source files directory are located at the # build directory since its already gitignore anyways. set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip) From d4ff90f1203f332d59d030f8ccf1adba6bc578b1 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 16:21:21 +0300 Subject: [PATCH 10/18] debug sqlite cmake status --- src/test-suite/sanity_check.sh.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 66450d02..ee71a478 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -49,6 +49,8 @@ perform_vector_comparison() { SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt + else + echo "${BOLD}{RED}SQLITE is off or not set: (@ENABLE_SQLITE@) ${NC}" fi # if file prefix is p or f, run the first while loop, else, run the second while loop From d7b6e2454becb2308efbf6e8b364fc6d1f7e642f Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 16:34:33 +0300 Subject: [PATCH 11/18] update sqlite flag checks --- src/test-suite/sanity_check.sh.cmake | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index ee71a478..22cde3f4 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -1,5 +1,6 @@ BUILD=@LIB_LOC@ +ENABLE_SQLITE=@ENABLE_SQLITE@ EncodingType=$1 SO_FILE= @@ -45,12 +46,12 @@ perform_vector_comparison() { # SQLite specific variables. - if [[ "@ENABLE_SQLITE@" == "ON" ]]; then + if [[ "$ENABLE_SQLITE" == "ON" ]]; then SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt else - echo "${BOLD}{RED}SQLITE is off or not set: (@ENABLE_SQLITE@) ${NC}" + echo "$(tput bold)${RED}SQLITE is off or not set: ($ENABLE_SQLITE) ${NC}" fi # if file prefix is p or f, run the first while loop, else, run the second while loop @@ -62,7 +63,7 @@ perform_vector_comparison() { wait # SQLITE is currently only tested against the program (p) level - if [[ "@ENABLE_SQLITE@" == "ON" && "$FILE_PREFIX" == "p" ]]; then + if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null fi else @@ -95,7 +96,7 @@ perform_vector_comparison() { fi # SQLite tests only if its enabled - if [[ "@ENABLE_SQLITE@" == "ON" ]]; then + if [[ "$ENABLE_SQLITE" == "ON" ]]; then if [[ ! -e "$SQLITE_VIR" ]]; then echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" exit 1 From 616712e343b81ed366709dbff533da53b6c38270 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 16:47:29 +0300 Subject: [PATCH 12/18] re-order cmake commands --- src/test-suite/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index 9204ec47..382a165c 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -1,3 +1,5 @@ +option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON) + configure_file(sanity_check.sh.cmake sanity_check.sh @ONLY) file(COPY PE-benchmarks-llfiles-llvm17 DESTINATION ./) file(COPY oracle DESTINATION ./) @@ -9,13 +11,11 @@ configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) file(COPY test-lit.py DESTINATION ./) file(COPY test-ir2vec.lit DESTINATION ./) -option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON) - -# SQLite IR file is added along side other PE-benchmarks for this version set(SQLITE_LLVM_IR ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll ) +# SQLite IR file is added along side other PE-benchmarks for this version if(NOT EXISTS ${SQLITE_LLVM_IR} AND ENABLE_SQLITE) # The downloaded Zip file and source files directory are located at the # build directory since its already gitignore anyways. From 88688efe694f90c16fa2d51b58dc147113b762b0 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 16:48:12 +0300 Subject: [PATCH 13/18] debug sqlite success status --- src/test-suite/sanity_check.sh.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 22cde3f4..770696c1 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -50,8 +50,10 @@ perform_vector_comparison() { SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt + + echo -e "${BLUE}${BOLD} SQLITE is on status=($ENABLE_SQLITE). ${NC}" else - echo "$(tput bold)${RED}SQLITE is off or not set: ($ENABLE_SQLITE) ${NC}" + echo -e "${RED}${BOLD} SQLITE is off or not set: ($ENABLE_SQLITE) ${NC}" fi # if file prefix is p or f, run the first while loop, else, run the second while loop From df0b3c7fc21167577bbdfa6db18fd400fb8af9dc Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 17:05:42 +0300 Subject: [PATCH 14/18] debug missing embedding --- src/test-suite/sanity_check.sh.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 770696c1..8e8562d1 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -101,6 +101,7 @@ perform_vector_comparison() { if [[ "$ENABLE_SQLITE" == "ON" ]]; then if [[ ! -e "$SQLITE_VIR" ]]; then echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" + echo -e "Generated embedding file name: ${SQLITE_VIR}. \n $(ls -l)" exit 1 fi mv ${SQLITE_VIR} ${TEMP}/ From b1f49a0103eb9845ffb609fc3e6f5e95a94c9b4d Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 17:42:11 +0300 Subject: [PATCH 15/18] update --- src/test-suite/sanity_check.sh.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 8e8562d1..3882269c 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -101,7 +101,7 @@ perform_vector_comparison() { if [[ "$ENABLE_SQLITE" == "ON" ]]; then if [[ ! -e "$SQLITE_VIR" ]]; then echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" - echo -e "Generated embedding file name: ${SQLITE_VIR}. \n $(ls -l)" + echo -e "Generated embedding file name: ${SQLITE_VIR}. \n $(ls ./temp_FA_llvm17_p -l)" exit 1 fi mv ${SQLITE_VIR} ${TEMP}/ From 503ed68612b7d201f545c3d598f26dd8554e2053 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 20:21:24 +0300 Subject: [PATCH 16/18] debug ir2vec command --- src/test-suite/sanity_check.sh.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index 3882269c..e1224fe1 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -66,7 +66,9 @@ perform_vector_comparison() { # SQLITE is currently only tested against the program (p) level if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then - ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null + echo -e "started generating embedding for SQLite benchmark." + ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} + echo -e "...embedding generated at: ${SQLITE_VIR}" fi else while IFS= read -r d_on From 563423c910f730da318cd1fa67970594ec388a72 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 20:53:53 +0300 Subject: [PATCH 17/18] cleanup --- src/test-suite/sanity_check.sh.cmake | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index e1224fe1..aea01c6f 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -50,10 +50,6 @@ perform_vector_comparison() { SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt - - echo -e "${BLUE}${BOLD} SQLITE is on status=($ENABLE_SQLITE). ${NC}" - else - echo -e "${RED}${BOLD} SQLITE is off or not set: ($ENABLE_SQLITE) ${NC}" fi # if file prefix is p or f, run the first while loop, else, run the second while loop @@ -66,9 +62,7 @@ perform_vector_comparison() { # SQLITE is currently only tested against the program (p) level if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then - echo -e "started generating embedding for SQLite benchmark." - ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} - echo -e "...embedding generated at: ${SQLITE_VIR}" + ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null fi else while IFS= read -r d_on @@ -103,7 +97,6 @@ perform_vector_comparison() { if [[ "$ENABLE_SQLITE" == "ON" ]]; then if [[ ! -e "$SQLITE_VIR" ]]; then echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" - echo -e "Generated embedding file name: ${SQLITE_VIR}. \n $(ls ./temp_FA_llvm17_p -l)" exit 1 fi mv ${SQLITE_VIR} ${TEMP}/ From 500c041bff8575fdafa3d972b7fce48249b1b2a2 Mon Sep 17 00:00:00 2001 From: m-atalla Date: Mon, 8 Jul 2024 20:54:09 +0300 Subject: [PATCH 18/18] re-order cmake commands --- src/test-suite/CMakeLists.txt | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index 382a165c..335ac7f1 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -1,16 +1,5 @@ option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON) -configure_file(sanity_check.sh.cmake sanity_check.sh @ONLY) -file(COPY PE-benchmarks-llfiles-llvm17 DESTINATION ./) -file(COPY oracle DESTINATION ./) -file(COPY ../../vocabulary DESTINATION ./) -file(COPY index-llvm17.files DESTINATION ./) - - -configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) -file(COPY test-lit.py DESTINATION ./) -file(COPY test-ir2vec.lit DESTINATION ./) - set(SQLITE_LLVM_IR ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll ) @@ -76,3 +65,15 @@ if(NOT EXISTS ${SQLITE_LLVM_IR} AND ENABLE_SQLITE) message(STATUS "SQLite benchmark was generated at: ${SQLITE_LLVM_IR}") endif() endif() + +# sanity checks and lit configs +configure_file(sanity_check.sh.cmake sanity_check.sh @ONLY) +file(COPY PE-benchmarks-llfiles-llvm17 DESTINATION ./) +file(COPY oracle DESTINATION ./) +file(COPY ../../vocabulary DESTINATION ./) +file(COPY index-llvm17.files DESTINATION ./) + + +configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) +file(COPY test-lit.py DESTINATION ./) +file(COPY test-ir2vec.lit DESTINATION ./)