Llvm14config

.gitignore

@@ -3,3 +3,6 @@ build/
 __pycache__/
 .vscode
 experiments/*/output
+
+seed_embeddings/triplets.txt
+seed_embeddings/preprocessed/*

seed_embeddings/OpenKE/generate_embedding.py

@@ -21,9 +21,9 @@ def train(arg_conf):
     con.set_in_path(arg_conf.index_dir)
     con.set_work_threads(4)
     con.set_train_times(arg_conf.epoch)
-    con.set_nbatches(100)
+    con.set_nbatches(nbatches=arg_conf.nbatches)
     con.set_alpha(0.001)
-    con.set_margin(1.0)
+    con.set_margin(arg_conf.margin)
     con.set_bern(0)
     con.set_dimension(arg_conf.dim)
     con.set_ent_neg_rate(1)
@@ -84,14 +84,30 @@ def findRep(src, dest, ent):
         type=int,
         default=300,
     )
+    parser.add_argument(
+        "--nbatches",
+        dest="nbatches",
+        help="Number of batches",
+        required=False,
+        type=int,
+        default=100,
+    )
+    parser.add_argument(
+        "--margin",
+        dest="margin",
+        help="Margin",
+        required=False,
+        type=float,
+        default=1.0,
+    )
 
     arg_conf = parser.parse_args()
 
     outfilejson = train(arg_conf)
 
     seedfile = os.path.join(
         arg_conf.index_dir,
-        "seedEmbedding_{}E_{}D.txt".format(arg_conf.epoch, arg_conf.dim),
+        "embeddings/seedEmbedding_{}E_{}D_{}batches{}margin.txt".format(arg_conf.epoch, arg_conf.dim, arg_conf.nbatches, arg_conf.margin),
     )
 
     findRep(outfilejson, seedfile, arg_conf.index_dir)

seed_embeddings/OpenKE/openKE.yaml

@@ -0,0 +1,101 @@
+name: openKE
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - _tflow_select=2.1.0=gpu
+  - absl-py=1.3.0=py37h06a4308_0
+  - aiohttp=3.8.3=py37h5eee18b_0
+  - aiosignal=1.2.0=pyhd3eb1b0_0
+  - astor=0.8.1=py37h06a4308_0
+  - astunparse=1.6.3=py_0
+  - async-timeout=4.0.2=py37h06a4308_0
+  - asynctest=0.13.0=py_0
+  - attrs=22.1.0=py37h06a4308_0
+  - blas=1.0=mkl
+  - blinker=1.4=py37h06a4308_0
+  - brotlipy=0.7.0=py37h27cfd23_1003
+  - c-ares=1.19.0=h5eee18b_0
+  - ca-certificates=2023.05.30=h06a4308_0
+  - cachetools=4.2.2=pyhd3eb1b0_0
+  - certifi=2022.12.7=py37h06a4308_0
+  - cffi=1.15.1=py37h5eee18b_3
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.0.4=py37h06a4308_0
+  - cryptography=39.0.1=py37h9ce1e76_0
+  - cudatoolkit=10.1.243=h6bb024c_0
+  - cudnn=7.6.5=cuda10.1_0
+  - cupti=10.1.168=0
+  - fftw=3.3.9=h27cfd23_1
+  - flit-core=3.6.0=pyhd3eb1b0_0
+  - frozenlist=1.3.3=py37h5eee18b_0
+  - gast=0.3.3=py_0
+  - google-auth=2.6.0=pyhd3eb1b0_0
+  - google-auth-oauthlib=0.4.4=pyhd3eb1b0_0
+  - google-pasta=0.2.0=pyhd3eb1b0_0
+  - grpcio=1.42.0=py37hce63b2e_0
+  - h5py=2.10.0=py37hd6299e0_1
+  - hdf5=1.10.6=h3ffc7dd_1
+  - idna=3.4=py37h06a4308_0
+  - importlib-metadata=4.11.3=py37h06a4308_0
+  - intel-openmp=2021.4.0=h06a4308_3561
+  - keras-preprocessing=1.1.2=pyhd3eb1b0_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgfortran-ng=11.2.0=h00389a5_1
+  - libgfortran5=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libprotobuf=3.20.3=he621ea3_0
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - markdown=3.4.1=py37h06a4308_0
+  - markupsafe=2.1.1=py37h7f8727e_0
+  - mkl=2021.4.0=h06a4308_640
+  - mkl-service=2.4.0=py37h7f8727e_0
+  - mkl_fft=1.3.1=py37hd3c417c_0
+  - mkl_random=1.2.2=py37h51133e4_0
+  - multidict=6.0.2=py37h5eee18b_0
+  - ncurses=6.4=h6a678d5_0
+  - numpy=1.21.5=py37h6c91a56_3
+  - numpy-base=1.21.5=py37ha15fc14_3
+  - oauthlib=3.2.1=py37h06a4308_0
+  - openssl=1.1.1t=h7f8727e_0
+  - opt_einsum=3.3.0=pyhd3eb1b0_1
+  - pip=22.3.1=py37h06a4308_0
+  - protobuf=3.20.3=py37h6a678d5_0
+  - pyasn1=0.4.8=pyhd3eb1b0_0
+  - pyasn1-modules=0.2.8=py_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyjwt=2.4.0=py37h06a4308_0
+  - pyopenssl=23.0.0=py37h06a4308_0
+  - pysocks=1.7.1=py37_1
+  - python=3.7.16=h7a1cb2a_0
+  - readline=8.2=h5eee18b_0
+  - requests=2.28.1=py37h06a4308_0
+  - requests-oauthlib=1.3.0=py_0
+  - rsa=4.7.2=pyhd3eb1b0_1
+  - scipy=1.7.3=py37h6c91a56_2
+  - setuptools=65.6.3=py37h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - sqlite=3.41.2=h5eee18b_0
+  - tensorboard=2.10.0=py37h06a4308_0
+  - tensorboard-data-server=0.6.1=py37h52d8a92_0
+  - tensorboard-plugin-wit=1.8.1=py37h06a4308_0
+  - tensorflow=2.2.0=gpu_py37h1a511ff_0
+  - tensorflow-base=2.2.0=gpu_py37h8a81be8_0
+  - tensorflow-estimator=2.6.0=pyh7b7c402_0
+  - tensorflow-gpu=2.2.0=h0d30ee6_0
+  - termcolor=2.1.0=py37h06a4308_0
+  - tk=8.6.12=h1ccaba5_0
+  - typing-extensions=4.4.0=py37h06a4308_0
+  - typing_extensions=4.4.0=py37h06a4308_0
+  - urllib3=1.26.14=py37h06a4308_0
+  - werkzeug=2.2.2=py37h06a4308_0
+  - wheel=0.38.4=py37h06a4308_0
+  - wrapt=1.14.1=py37h5eee18b_0
+  - xz=5.4.2=h5eee18b_0
+  - yarl=1.8.1=py37h5eee18b_0
+  - zipp=3.11.0=py37h06a4308_0
+  - zlib=1.2.13=h5eee18b_0
+prefix: /home/intern23002/anaconda3/envs/openKE

seed_embeddings/OpenKE/requirements.txt

@@ -3,100 +3,101 @@
 # platform: linux-64
 _libgcc_mutex=0.1=main
 _tflow_select=2.3.0=mkl
-absl-py=0.10.0=py37_0
-aiohttp=3.7.4=py37h27cfd23_1
-appdirs=1.4.4=pyh9f0ad1d_0
-astor=0.8.1=py37_0
-astunparse=1.6.3=py_0
-async-timeout=3.0.1=py37h06a4308_0
-attrs=20.3.0=pyhd3eb1b0_0
-blas=1.0=mkl
-blinker=1.4=py37h06a4308_0
-brotlipy=0.7.0=py37h27cfd23_1003
-c-ares=1.16.1=h7b6447c_0
-ca-certificates=2020.12.5=ha878542_0
-cachetools=4.2.1=pyhd3eb1b0_0
-certifi=2020.12.5=py37h89c1867_1
-cffi=1.14.5=py37h261ae71_0
-cfgv=3.2.0=py_0
-chardet=3.0.4=py37h06a4308_1003
-click=7.1.2=pyhd3eb1b0_0
-cryptography=3.3.2=py37h3c74f83_0
-distlib=0.3.1=pyh9f0ad1d_0
-editdistance=0.5.3=py37h3340039_2
-filelock=3.0.12=pyh9f0ad1d_0
-gast=0.3.3=py_0
-google-auth=1.27.1=pyhd3eb1b0_0
-google-auth-oauthlib=0.4.3=pyhd3eb1b0_0
-google-pasta=0.2.0=py_0
-grpcio=1.31.0=py37hf8bcb03_0
-h5py=2.10.0=py37hd6299e0_1
-hdf5=1.10.6=hb1b8bf9_0
-identify=2.1.0=pyhd8ed1ab_0
-idna=2.10=pyhd3eb1b0_0
-importlib-metadata=2.0.0=py_1
+absl-py=0.10.0
+aiohttp=3.7.4
+appdirs=1.4.4
+astor=0.8.1
+astunparse=1.6.3
+async-timeout=3.0.1
+attrs=20.3.0
+blas=1.0
+blinker=1.4
+brotlipy=0.7.0
+c-ares=1.16.1
+# ca-certificates=2020.12.5
+cachetools=4.2.1
+certifi=2020.12.5
+cffi=1.14.5
+# cfgv=3.2.0
+chardet=3.0.4
+click=7.1.2
+cryptography=3.3.2
+distlib=0.3.1
+editdistance=0.5.3
+filelock=3.0.12
+gast=0.3.3
+google-auth=1.27.1
+google-auth-oauthlib=0.4.3
+google-pasta=0.2.0
+grpcio=1.31.0
+h5py=2.10.0
+hdf5=1.10.6
+# identify=2.1.0
+idna=2.10
+importlib-metadata=2.0.0
 importlib_metadata=2.0.0=1
 intel-openmp=2020.2=254
-keras-applications=1.0.8=py_1
-keras-preprocessing=1.1.0=py_1
-ld_impl_linux-64=2.33.1=h53a641e_7
-libedit=3.1.20191231=h14c3975_1
-libffi=3.3=he6710b0_2
-libgcc-ng=9.1.0=hdf63c60_0
-libgfortran-ng=7.3.0=hdf63c60_0
-libprotobuf=3.13.0=hd408876_0
-libstdcxx-ng=9.1.0=hdf63c60_0
-markdown=3.3=py37_0
+keras-applications=1.0.8
+keras-preprocessing=1.1.0
+ld_impl_linux-64=2.33.1
+libedit=3.1.20191231
+libffi=3.3
+libgcc-ng=9.1.0
+libgfortran-ng=7.3.0
+libprotobuf=3.13.0
+libstdcxx-ng=9.1.0
+markdown=3.3
 mkl=2020.2=256
-mkl-service=2.3.0=py37he904b0f_0
-mkl_fft=1.2.0=py37h23d657b_0
-mkl_random=1.1.1=py37h0573a6f_0
-mock=4.0.2=py_0
-multidict=5.1.0=py37h27cfd23_2
-ncurses=6.2=he6710b0_1
-nodeenv=1.5.0=pyh9f0ad1d_0
-numpy=1.19.1=py37hbc911f0_0
-numpy-base=1.19.1=py37hfa32c7d_0
-oauthlib=3.1.0=py_0
-openssl=1.1.1k=h27cfd23_0
-opt_einsum=3.1.0=py_0
-pip=20.2.3=py37_0
-pre-commit=2.9.3=py37h89c1867_0
-protobuf=3.13.0=py37hf484d3e_1
-pyasn1=0.4.8=py_0
-pyasn1-modules=0.2.8=py_0
-pycparser=2.20=py_2
-pyjwt=1.7.1=py37_0
-pyopenssl=20.0.1=pyhd3eb1b0_1
-pysocks=1.7.1=py37_1
-python=3.7.9=h7579374_0
-python_abi=3.7=1_cp37m
-pyyaml=5.4.1=py37h27cfd23_1
-readline=8.0=h7b6447c_0
-requests=2.25.1=pyhd3eb1b0_0
-requests-oauthlib=1.3.0=py_0
-rsa=4.7.2=pyhd3eb1b0_1
-scipy=1.5.2=py37h0b6359f_0
-setuptools=50.3.0=py37hb0f4dca_1
-six=1.15.0=py_0
-sqlite=3.33.0=h62c20be_0
-tensorboard=2.4.0=pyhc547734_0
-tensorboard-plugin-wit=1.6.0=py_0
-tensorflow=2.2.0=mkl_py37h6e9ce2d_0
-tensorflow-base=2.2.0=mkl_py37hd506778_0
-tensorflow-estimator=2.2.0=pyh208ff02_0
-termcolor=1.1.0=py37_1
-tk=8.6.10=hbc83047_0
-toml=0.10.2=pyhd8ed1ab_0
-typing-extensions=3.7.4.3=hd3eb1b0_0
-typing_extensions=3.7.4.3=pyh06a4308_0
-urllib3=1.26.5=pyhd8ed1ab_0
-virtualenv=20.4.2=py37h89c1867_0
-werkzeug=1.0.1=py_0
-wheel=0.35.1=py_0
-wrapt=1.12.1=py37h7b6447c_1
-xz=5.2.5=h7b6447c_0
-yaml=0.2.5=h516909a_0
-yarl=1.6.3=py37h27cfd23_0
-zipp=3.3.0=py_0
-zlib=1.2.11=h7b6447c_3
+mkl-service=2.3.0
+mkl_fft=1.2.0
+mkl_random=1.1.1
+mock=4.0.2
+multidict=5.1.0
+ncurses=6.2
+# nodeenv=1.5.0
+numpy=1.19.1
+numpy-base=1.19.1
+oauthlib=3.1.0
+openssl=1.1.1k
+opt_einsum=3.1.0
+pip=20.2.3
+# pre-commit=2.9.3f
+protobuf=3.13.0
+pyasn1=0.4.8
+pyasn1-modules=0.2.8
+pycparser=2.20
+pyjwt=1.7.1
+pyopenssl=20.0.1
+pysocks=1.7.1
+python=3.7.9
+# python_abi=3.7=1_cp37m
+pyyaml=5.4.1
+readline=8.0
+requests=2.25.1
+requests-oauthlib=1.3.0
+rsa=4.7.2
+scipy=1.5.2
+setuptools=50.3.0
+six=1.15.0
+sqlite=3.33.0
+tensorboard=2.4.0
+tensorboard-plugin-wit=1.6.0
+# tensorflow=2.2.0=mkl_py37h6e9ce2d_0
+tensorflow-gpu
+tensorflow-base=2.2.0
+tensorflow-estimator=2.2.0
+termcolor=1.1.0
+tk=8.6.10
+toml=0.10.2
+typing-extensions=3.7.4.3
+typing_extensions=3.7.4.3
+# urllib3=1.26.5
+virtualenv=20.4.2
+werkzeug=1.0.1
+wheel=0.35.1
+wrapt=1.12.1
+xz=5.2.5
+yaml=0.2.5
+yarl=1.6.3
+zipp=3.3.0
+zlib=1.2.11

seed_embeddings/triplets.sh

@@ -45,7 +45,7 @@ fi
 counter=0
 while read p; do
 	let "counter++"
-	DEBUG echo "collecting data from $p"
+	echo "collecting data from $p"
 	NO_OF_OPT_FILES=$2
 	DEBUG echo "NO_OF_OPT_FILES from $NO_OF_OPT_FILES"
 	OPT_LEVELS=("O0" "O1" "O2" "O3" "Os" "Oz")
@@ -69,18 +69,18 @@ while read p; do
 		fi
 		USED_OPT[$a]=$opt
 		DEBUG echo "opt from $opt"
-		${LLVM_BUILD}/bin/opt-12 -S -$opt $p -o $tmpfile
+		${LLVM_BUILD}/bin/opt-14 -S -$opt $p -o $tmpfile
 		$COLLECT_BUILD/bin/ir2vec -collectIR -o $4 $tmpfile &>/dev/null
 		let "a++"
 		rm "$tmpfile"
-	done &
-	if [ $counter == 100 ]; then
-		sleep 20
-		counter=0
-	fi
 
+		# sleep 2
+	done
 done <$3
 
 wait
+
+# print counter
+echo $counter
 echo $4
 echo "All ll files processed"