Skip to content

Commit

Permalink
Merge pull request #113 from m-atalla/sqlite-benchmark
Browse files Browse the repository at this point in the history
Adding SQLite as part of PE Benchmark test runs
  • Loading branch information
svkeerthy authored Jul 9, 2024
2 parents d8a63df + 500c041 commit dd38800
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 0 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ seed_embeddings/preprocessed/*

html
latex
sqlite3.ll
69 changes: 69 additions & 0 deletions src/test-suite/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,72 @@
option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON)

set(SQLITE_LLVM_IR
${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll
)

# SQLite IR file is added along side other PE-benchmarks for this version
if(NOT EXISTS ${SQLITE_LLVM_IR} AND ENABLE_SQLITE)
# The downloaded Zip file and source files directory are located at the
# build directory since its already gitignore anyways.
set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip)
set(SQLITE_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000)

# Some of the recommended compile flags. https://sqlite.org/amalgamation.html
# with the following flags the resulting LLVM IR file is about 500k lines
set(SQLITE_COMPILE_FLAGS
-DSQLITE_ENABLE_FTS3
-DSQLITE_ENABLE_FTS5
-DSQLITE_ENABLE_RTREE
-DSQLITE_ENABLE_DBSTAT_VTAB
-DSQLITE_ENABLE_RBU
-DSQLITE_ENABLE_SESSION
)

if(NOT EXISTS ${SQLITE_ZIP})
# SQLite Amalgamation download source: https://sqlite.org/download.html
message(STATUS "Downloading SQLite Amalgamation source zip file..")
file(
DOWNLOAD https://sqlite.org/2024/sqlite-amalgamation-3460000.zip
${SQLITE_ZIP}
EXPECTED_HASH SHA3_256=1221eed70de626871912bfca144c00411f0c30d3c2b7935cff3963b63370ef7c
SHOW_PROGRESS
)
endif()

if(NOT EXISTS ${SQLITE_SOURCE})
message(STATUS
"Unzipping SQLite source files at: ${SQLITE_SOURCE}"
)
execute_process(
COMMAND unzip ${SQLITE_ZIP}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE unzip_result
)

if(NOT unzip_result EQUAL 0)
message(FATAL_ERROR "Failed to extract SQLite zip file")
endif()
endif()

find_program(CLANG_BIN NAMES clang-17 clang)
if(NOT CLANG_BIN)
message(FATAL_ERROR "Clang not found. Please install it.")
endif()

execute_process(
COMMAND ${CLANG_BIN} -S -emit-llvm ${SQLITE_COMPILE_FLAGS}
-Xclang -disable-O0-optnone
-o ${SQLITE_LLVM_IR}
${SQLITE_SOURCE}/sqlite3.c
RESULT_VARIABLE sqlite_ir_result
)

if(sqlite_ir_result EQUAL 0)
message(STATUS "SQLite benchmark was generated at: ${SQLITE_LLVM_IR}")
endif()
endif()

# sanity checks and lit configs
configure_file(sanity_check.sh.cmake sanity_check.sh @ONLY)
file(COPY PE-benchmarks-llfiles-llvm17 DESTINATION ./)
file(COPY oracle DESTINATION ./)
Expand Down
1 change: 1 addition & 0 deletions src/test-suite/oracle/FA_llvm17_p/sqlite3.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
39389.228010 -5376.674414 -29021.478687 57311.999476 -2830.058969 -4903.917802 12531.761407 11238.552214 5837.613706 -75945.222073 -29524.528980 67012.326909 12066.566763 -3894.166052 -5155.274300 24880.990511 16379.301698 18119.941570 -78168.641123 6989.300529 12382.527812 -4140.765868 -22445.730997 -11017.107544 -2946.210374 -24964.022127 31784.060329 24619.920816 3814.250199 53991.318974 -15070.562586 30707.538848 43953.849707 -2381.574458 22950.984229 41734.144641 -10816.221965 -4810.135903 -22139.721114 -18213.911622 65218.353935 28923.175315 -4972.659720 25150.057632 -1473.603306 60679.434496 14821.739239 48281.290493 15413.638264 -21271.516911 17702.110667 29020.863980 -31464.468252 -12264.696262 -53225.776560 4551.504887 20417.821305 12091.340519 -24138.628052 28349.748690 -17150.800330 12706.377159 24665.181318 16145.634347 65299.241720 21431.810517 40498.453896 51106.281433 54662.926106 -40088.949771 13304.092762 7654.656533 14419.047690 1069.853817 48025.749385 -22396.022974 -42850.026510 -6809.055547 -521.476994 -64375.753597 -7862.747134 -5017.845790 81997.642405 -46507.572785 -7365.529276 1302.849919 25304.077449 58076.845632 -34610.699881 4141.103685 -51849.889573 -3722.191701 -9419.574215 79247.276286 9473.697538 19713.967952 -12465.907796 19244.744645 -3435.023306 13989.461256 6536.506768 -56492.493102 -1688.045743 -73137.700853 -29253.134499 -40407.956653 -933.743462 14279.813178 -57154.836462 557.310850 4767.098763 53947.134808 -84054.927478 -5960.262662 15618.135863 6821.010998 52393.624293 12726.230237 -20777.351490 9294.755970 1390.254836 17729.270299 23544.548798 -26589.157997 11527.892553 -2477.706692 2599.986765 10135.742292 17323.593387 18006.984562 -84040.870641 -13220.164514 23623.495802 -7462.858326 7428.499912 29600.097469 25956.154793 -22568.528168 22848.602044 -41683.667110 46822.194460 -63180.261022 -16181.937610 -23577.152061 -38918.014782 -27130.354619 40871.462790 45417.167782 45708.189488 -8005.036744 -9525.828909 -574.773616 -72716.434262 -90806.812223 10465.695405 -1704.417641 -3791.325645 11437.562356 -9734.447109 10740.288521 -43880.468152 14900.821761 34527.372778 -29568.994163 -26237.248706 -3183.854486 44694.863175 10560.089449 -18749.784008 -15393.422626 -7807.926404 -3875.797153 23007.442336 54734.520268 -18045.888336 -91677.309373 -81314.052401 -25529.987834 49294.973712 -25441.928778 -4832.642230 -21895.379659 -25103.559014 -31507.116894 -2475.733488 -39250.694689 12937.243692 -6512.779912 -17903.734550 -64470.347896 22094.732030 -9237.200299 -14267.520417 10192.716821 -47076.708383 10964.664559 30742.866710 4235.252363 1191.728237 10027.339188 4415.971911 -38850.183036 -29519.167851 20492.481769 -23659.175199 52721.378257 42300.583697 2340.791768 -29988.064784 -19580.261322 -49547.471529 33755.555148 -48905.776848 35422.181670 8024.147986 -27027.556318 33921.390322 -44944.517675 -7937.652834 -29487.218742 26365.878881 6197.248727 -71044.945394 70024.038121 51335.191099 -42336.803306 -3397.896498 -9550.309290 23323.897056 11667.371271 39492.749377 -43755.789585 -36460.205077 14610.025897 151.649603 -1885.533970 -2971.605785 14692.409094 -12728.623378 -2336.804940 -48256.526733 -15118.845499 47250.475424 26403.952496 -6263.492780 1761.811526 13312.365912 -25423.677786 12047.566917 49352.438572 30559.417573 12879.122901 -45558.117241 -102363.066526 21858.284461 -27296.611234 -9673.486010 6833.930802 -27911.443354 -18580.310923 16977.709576 -28889.467960 48066.249561 -17385.422090 -33583.855529 46193.509969 -69949.758441 -54972.692493 -46011.694699 -13363.262082 11685.615563 -4471.066728 22318.769605 -21219.305738 9400.389388 -7905.007664 19022.545764 -19689.155351 -29574.226282 12494.780265 28997.563923 45806.675927 7127.876997 -14628.763346 57523.118124 9747.445151 -12639.944425 -14974.659152 -5337.028753 -25489.188876 8811.413006 -66892.993020 -10358.495763 4699.510652 33516.701552 -40010.545447 22338.301792 70.999160 17700.864559 10359.928443
1 change: 1 addition & 0 deletions src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
16914.214172 1840.963612 -8056.768579 21579.305453 -906.262205 -14727.750711 4137.897728 12411.245494 -6586.577729 -37492.964734 -23503.579107 24019.665282 3178.140591 -4723.500753 5407.842244 7912.921167 -3458.007608 3013.286848 -43144.929418 15115.546246 13531.053632 4908.258806 -5820.302126 -7871.660645 3889.551273 -10608.906240 1448.299046 10127.834420 1014.800933 22875.036123 -11217.396606 7132.652507 18359.679961 -541.432431 9754.327113 19447.189792 -5458.297947 1001.803515 1260.266054 429.515246 30198.629348 18262.393044 -3520.028532 10885.129589 9038.867032 23509.461660 -2190.924487 20668.772705 -4464.237518 -15003.251139 10138.776080 7383.103845 -17749.750384 -2971.023222 -14480.273423 9624.178060 19172.138324 8872.303230 -11147.777716 11520.362299 -11789.698651 -2464.035909 7416.324752 381.487262 27537.998172 14634.787225 15946.290784 14708.178722 30142.986762 -11951.156505 -2847.544498 13036.279684 11946.873288 -2281.002892 14178.835480 -9445.563955 -21959.072204 -11427.626110 14.950048 -38164.969681 -15326.407916 -604.005724 40969.720254 -22921.026771 1021.245459 6482.292367 8335.949495 22709.037213 -15671.772052 -634.916992 -16388.880672 -88.372465 -10411.435811 38141.528995 1444.237535 7092.407774 -3015.810814 9673.987822 10480.803898 17367.615576 -5099.230566 -15713.135321 -16277.581293 -37081.163429 -8470.125739 -8772.651484 3261.668365 7589.841179 -31083.117699 -6787.744450 7673.834177 27727.294920 -34264.453388 -1024.019126 18412.358452 9384.182441 16855.866440 7579.575966 -20625.224866 4570.472021 1587.196202 1714.091642 13996.430750 -8729.059297 -1445.531903 -5345.869665 6737.841282 747.071812 5712.435997 1316.571824 -37393.893115 -11137.285625 3457.923217 -7597.644020 934.439596 4310.015852 8189.372290 -13761.423625 12699.496546 -14782.421306 18183.527744 -33691.703352 -517.262789 -11239.399834 -19974.966452 -3504.060581 16885.975472 25761.155805 14053.165475 -6681.317758 -6153.418469 -7325.079859 -30356.658229 -34656.871696 6283.765911 -1758.163001 8988.464773 -1522.551626 -1673.995728 5009.828211 -9656.368676 1426.665633 9335.348034 -11649.556808 -1566.318860 -6295.734684 31544.197891 661.649733 -4830.674434 -14478.650575 -8427.805505 -545.973874 2502.690463 15002.196104 -6300.676337 -33845.085246 -34194.679896 -18256.095681 19120.451916 -4796.204741 -1357.198324 -13606.642437 -3021.046722 -18445.044067 3332.247063 -17445.313795 6821.379552 -1561.129509 -13353.268278 -23115.474322 472.936692 -1898.996769 -9484.504976 -5017.053747 -15333.591479 -5749.949470 18458.335412 4047.862621 8084.542286 3028.814311 1129.188741 -21428.210033 -19353.038396 5786.410664 -7532.951615 21430.806942 19577.147353 459.479687 -18445.971763 -4433.705096 -27620.540913 19828.835893 -17033.978568 4474.063320 -6138.081727 -9289.859715 29276.565507 -13631.410108 -7187.016573 -18083.697502 8781.137185 3854.077762 -34757.682776 32445.021522 20843.803376 -11541.272980 -4382.951260 -1828.406638 2398.068724 18359.047156 21539.259133 -15484.985073 -13341.607581 3250.685126 -6791.923469 -1860.182208 -395.840937 16642.290996 -16803.371117 11532.507834 -15286.484050 -9243.681272 12537.733834 16148.825960 493.864148 8958.987453 -833.191497 -2730.725876 8574.123858 17163.311834 13874.951473 -6250.890749 -19190.295017 -44012.399335 13558.447731 -8566.129714 -6785.466744 -403.105960 -9429.032390 -19264.410796 3993.546581 -5802.231716 24354.598056 802.353023 -11928.318532 16617.764262 -27415.844372 -24580.951602 -13247.186538 2753.917039 -291.838695 -5530.368520 7072.636033 -7053.490037 6451.836519 9084.519734 856.179694 1720.038980 -17071.204665 -10241.571031 -236.670560 18695.744940 8736.999602 -4602.100388 24403.244235 5754.482284 -10343.410231 -6480.548159 858.463332 31.853096 -2285.052285 -20497.891846 -9698.522648 -9221.781745 8000.340915 -9367.694417 14972.296732 1337.978384 1709.723559 9941.290745
33 changes: 33 additions & 0 deletions src/test-suite/sanity_check.sh.cmake
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

BUILD=@LIB_LOC@
ENABLE_SQLITE=@ENABLE_SQLITE@
EncodingType=$1

SO_FILE=
Expand Down Expand Up @@ -43,13 +44,26 @@ perform_vector_comparison() {
ORIG_FILE=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/ir2vec.txt
VIR_FILE=ir2vec_${FILE_PREFIX}.txt


# SQLite specific variables.
if [[ "$ENABLE_SQLITE" == "ON" ]]; then
SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt
SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll
SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt
fi

# if file prefix is p or f, run the first while loop, else, run the second while loop

if [[ "$FILE_PREFIX" == "p" || "$FILE_PREFIX" == "f" ]]; then
while IFS= read -r d; do
${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${VIR_FILE} ${d} &> /dev/null
done < index-${SEED_VERSION}.files
wait

# SQLITE is currently only tested against the program (p) level
if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then
${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null
fi
else
while IFS= read -r d_on
do
Expand Down Expand Up @@ -78,6 +92,25 @@ perform_vector_comparison() {
echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}"
exit 1
fi

# SQLite tests only if its enabled
if [[ "$ENABLE_SQLITE" == "ON" ]]; then
if [[ ! -e "$SQLITE_VIR" ]]; then
echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}"
exit 1
fi
mv ${SQLITE_VIR} ${TEMP}/

d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${SQLITE_ORIG}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${SQLITE_VIR}))

if [ "$d" == "" ]; then
echo -e "${GREEN}${BOLD}[Test Passed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}"
else
echo -e "$(tput bold)${RED}[Test Failed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}"
exit 1
fi
fi

else
if ls *${VIR_FILE} 1> /dev/null 2>&1
then
Expand Down

0 comments on commit dd38800

Please sign in to comment.