diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index 62ed12d856c..57a925667d5 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -199,7 +199,13 @@ public class CommonParameter { //If you are running a solidity node for java tron, this flag is set to true @Getter @Setter + @Parameter(names = {"--solidity"}, description = "running a solidity node for java tron") public boolean solidityNode = false; + //If you are running KeystoreFactory, this flag is set to true + @Getter + @Setter + @Parameter(names = {"--keystore"}, description = "running KeystoreFactory") + public boolean keystore = false; @Getter @Setter public int rpcPort; diff --git a/framework/build.gradle b/framework/build.gradle index ec113c93cb1..c182aa7431f 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -8,6 +8,7 @@ gitProperties.failOnNoGitDirectory = false; apply plugin: 'application' apply plugin: 'checkstyle' +apply plugin: 'com.github.johnrengelman.shadow' mainClassName = 'org.tron.program.FullNode' @@ -150,32 +151,6 @@ jacocoTestReport { getExecutionData().setFrom(fileTree('../framework/build/jacoco').include("**.exec")) } -def binaryRelease(taskName, jarName, mainClass) { - return tasks.create("${taskName}", Jar) { - baseName = jarName - version = null - from(sourceSets.main.output) { - include "/**" - } - - from { - configurations.runtimeClasspath.collect { - it.isDirectory() ? it : zipTree(it) - } - } - - // exclude these files for bouncycastle - exclude "META-INF/*.SF" - exclude "META-INF/*.DSA" - exclude "META-INF/*.RSA" - - manifest { - attributes "Main-Class": "${mainClass}" - } - -// classifier(jarName) - } -} def createScript(project, mainClass, name) { project.tasks.create(name: name, type: CreateStartScripts) { @@ -217,35 +192,24 @@ startScripts.enabled = false run.enabled = false tasks.distTar.enabled = false -createScript(project, 'org.tron.program.SolidityNode', 'SolidityNode') createScript(project, 'org.tron.program.FullNode', 'FullNode') -createScript(project, 'org.tron.program.KeystoreFactory', 'KeystoreFactory') -createScript(project, 'org.tron.program.DBConvert', 'DBConvert') - -def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' -def skipSolidity = hasProperty('skipSolidity') ? true : false -def skipKeystore = hasProperty('skipKeystore') ? true : false -def skipConvert = hasProperty('skipConvert') ? true : false -def skipAll = hasProperty('skipAll') ? true : false -if (releaseBinary == 'true') { - artifacts { - archives(binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode')) - } - if (!skipAll) { - if (!skipSolidity) { - artifacts { - archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'))} - } - if (!skipKeystore) { - artifacts { - archives(binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'))} - } - if (!skipConvert) { - artifacts { - archives(binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'))} - } - } +shadowJar { + baseName = 'FullNode' + classifier = null + version = null + // exclude these files for bouncycastle + exclude "META-INF/*.SF" + exclude "META-INF/*.DSA" + exclude "META-INF/*.RSA" + mergeServiceFiles() // https://github.com/grpc/grpc-java/issues/10853 +} +shadowDistZip { + enabled = false +} + +startShadowScripts { + enabled = false } task copyToParent(type: Copy) { diff --git a/framework/src/main/java/org/tron/program/DBConvert.java b/framework/src/main/java/org/tron/program/DBConvert.java deleted file mode 100644 index a13a2ffefb5..00000000000 --- a/framework/src/main/java/org/tron/program/DBConvert.java +++ /dev/null @@ -1,412 +0,0 @@ -package org.tron.program; - -import static org.fusesource.leveldbjni.JniDBFactory.factory; - -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.fusesource.leveldbjni.JniDBFactory; -import org.iq80.leveldb.CompressionType; -import org.iq80.leveldb.DB; -import org.iq80.leveldb.DBIterator; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; -import org.rocksdb.ComparatorOptions; -import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.Status; -import org.tron.common.utils.FileUtil; -import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; -import org.tron.common.utils.MarketOrderPriceComparatorForRockDB; -import org.tron.common.utils.PropUtil; - -@Slf4j -public class DBConvert implements Callable { - - static { - RocksDB.loadLibrary(); - } - - private final String srcDir; - private final String dstDir; - private final String dbName; - private final Path srcDbPath; - private final Path dstDbPath; - - private long srcDbKeyCount = 0L; - private long dstDbKeyCount = 0L; - private long srcDbKeySum = 0L; - private long dstDbKeySum = 0L; - private long srcDbValueSum = 0L; - private long dstDbValueSum = 0L; - private final long startTime; - private static final int CPUS = Runtime.getRuntime().availableProcessors(); - private static final int BATCH = 256; - private static final String CHECKPOINT_V2_DIR_NAME = "checkpoint"; - - - @Override - public Boolean call() throws Exception { - return doConvert(); - } - - public DBConvert(String src, String dst, String name) { - this.srcDir = src; - this.dstDir = dst; - this.dbName = name; - this.srcDbPath = Paths.get(this.srcDir, name); - this.dstDbPath = Paths.get(this.dstDir, name); - this.startTime = System.currentTimeMillis(); - } - - public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { - org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); - dbOptions.createIfMissing(true); - dbOptions.paranoidChecks(true); - dbOptions.verifyChecksums(true); - dbOptions.compressionType(CompressionType.SNAPPY); - dbOptions.blockSize(4 * 1024); - dbOptions.writeBufferSize(10 * 1024 * 1024); - dbOptions.cacheSize(10 * 1024 * 1024L); - dbOptions.maxOpenFiles(1000); - return dbOptions; - } - - public static void main(String[] args) { - int code = run(args); - logger.info("exit code {}.", code); - System.out.printf("exit code %d.\n", code); - System.exit(code); - } - - public static int run(String[] args) { - String dbSrc; - String dbDst; - if (args.length < 2) { - dbSrc = "output-directory/database"; - dbDst = "output-directory-dst/database"; - } else { - dbSrc = args[0]; - dbDst = args[1]; - } - File dbDirectory = new File(dbSrc); - if (!dbDirectory.exists()) { - logger.info(" {} does not exist.", dbSrc); - return 404; - } - List files = Arrays.stream(Objects.requireNonNull(dbDirectory.listFiles())) - .filter(File::isDirectory) - .filter(e -> !CHECKPOINT_V2_DIR_NAME.equals(e.getName())) - .collect(Collectors.toList()); - - // add checkpoint v2 convert - File cpV2Dir = new File(Paths.get(dbSrc, CHECKPOINT_V2_DIR_NAME).toString()); - List cpList = null; - if (cpV2Dir.exists()) { - cpList = Arrays.stream(Objects.requireNonNull(cpV2Dir.listFiles())) - .filter(File::isDirectory) - .collect(Collectors.toList()); - } - - if (files.isEmpty()) { - logger.info("{} does not contain any database.", dbSrc); - return 0; - } - final long time = System.currentTimeMillis(); - final List> res = new ArrayList<>(); - - final ThreadPoolExecutor esDb = new ThreadPoolExecutor( - CPUS, 16 * CPUS, 1, TimeUnit.MINUTES, - new ArrayBlockingQueue<>(CPUS, true), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.CallerRunsPolicy()); - - esDb.allowCoreThreadTimeOut(true); - - files.forEach(f -> res.add(esDb.submit(new DBConvert(dbSrc, dbDst, f.getName())))); - // convert v2 - if (cpList != null) { - cpList.forEach(f -> res.add(esDb.submit( - new DBConvert(dbSrc + "/" + CHECKPOINT_V2_DIR_NAME, - dbDst + "/" + CHECKPOINT_V2_DIR_NAME, f.getName())))); - } - - int fails = res.size(); - - for (Future re : res) { - try { - if (re.get()) { - fails--; - } - } catch (InterruptedException e) { - logger.error("{}", e); - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - logger.error("{}", e); - } - } - - esDb.shutdown(); - logger.info("database convert use {} seconds total.", - (System.currentTimeMillis() - time) / 1000); - if (fails > 0) { - logger.error("failed!!!!!!!!!!!!!!!!!!!!!!!! size:{}", fails); - } - return fails; - } - - public DB newLevelDb(Path db) throws Exception { - DB database; - File file = db.toFile(); - org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); - if ("market_pair_price_to_order".equalsIgnoreCase(this.dbName)) { - dbOptions.comparator(new MarketOrderPriceComparatorForLevelDB()); - } - database = factory.open(file, dbOptions); - return database; - } - - private Options newDefaultRocksDbOptions() { - Options options = new Options(); - options.setCreateIfMissing(true); - options.setIncreaseParallelism(1); - options.setNumLevels(7); - options.setMaxOpenFiles(5000); - options.setTargetFileSizeBase(64 * 1024 * 1024); - options.setTargetFileSizeMultiplier(1); - options.setMaxBytesForLevelBase(512 * 1024 * 1024); - options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); - options.setLevel0FileNumCompactionTrigger(4); - options.setLevelCompactionDynamicLevelBytes(true); - if ("market_pair_price_to_order".equalsIgnoreCase(this.dbName)) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - final BlockBasedTableConfig tableCfg; - options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); - tableCfg.setBlockSize(64 * 1024); - tableCfg.setBlockCacheSize(32 * 1024 * 1024); - tableCfg.setCacheIndexAndFilterBlocks(true); - tableCfg.setPinL0FilterAndIndexBlocksInCache(true); - tableCfg.setFilter(new BloomFilter(10, false)); - options.prepareForBulkLoad(); - return options; - } - - public RocksDB newRocksDb(Path db) { - RocksDB database = null; - try (Options options = newDefaultRocksDbOptions()) { - database = RocksDB.open(options, db.toString()); - } catch (Exception e) { - logger.error("{}", e); - } - return database; - } - - private void batchInsert(RocksDB rocks, List keys, List values) - throws Exception { - try (org.rocksdb.WriteBatch batch = new org.rocksdb.WriteBatch()) { - for (int i = 0; i < keys.size(); i++) { - byte[] k = keys.get(i); - byte[] v = values.get(i); - batch.put(k, v); - } - write(rocks, batch); - } - keys.clear(); - values.clear(); - } - - /** - * https://github.com/facebook/rocksdb/issues/6625 - * @param rocks db - * @param batch write batch - * @throws Exception RocksDBException - */ - private void write(RocksDB rocks, org.rocksdb.WriteBatch batch) throws Exception { - try { - rocks.write(new org.rocksdb.WriteOptions(), batch); - } catch (RocksDBException e) { - // retry - if (maybeRetry(e)) { - TimeUnit.MILLISECONDS.sleep(1); - write(rocks, batch); - } else { - throw e; - } - } - } - - private boolean maybeRetry(RocksDBException e) { - boolean retry = false; - if (e.getStatus() != null) { - retry = e.getStatus().getCode() == Status.Code.TryAgain - || e.getStatus().getCode() == Status.Code.Busy - || e.getStatus().getCode() == Status.Code.Incomplete; - } - return retry || (e.getMessage() != null && ("Write stall".equalsIgnoreCase(e.getMessage()) - || ("Incomplete").equalsIgnoreCase(e.getMessage()))); - } - - /** - * https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ . - * What's the fastest way to load data into RocksDB? - * @param level leveldb - * @param rocks rocksdb - * @return if ok - */ - public boolean convertLevelToRocksBatchIterator(DB level, RocksDB rocks) { - // convert - List keys = new ArrayList<>(BATCH); - List values = new ArrayList<>(BATCH); - try (DBIterator levelIterator = level.iterator( - new org.iq80.leveldb.ReadOptions().fillCache(false))) { - - JniDBFactory.pushMemoryPool(1024 * 1024); - levelIterator.seekToFirst(); - - while (levelIterator.hasNext()) { - Map.Entry entry = levelIterator.next(); - byte[] key = entry.getKey(); - byte[] value = entry.getValue(); - srcDbKeyCount++; - srcDbKeySum = byteArrayToIntWithOne(srcDbKeySum, key); - srcDbValueSum = byteArrayToIntWithOne(srcDbValueSum, value); - keys.add(key); - values.add(value); - if (keys.size() >= BATCH) { - try { - batchInsert(rocks, keys, values); - } catch (Exception e) { - logger.error("{}", e); - return false; - } - } - } - - if (!keys.isEmpty()) { - try { - batchInsert(rocks, keys, values); - } catch (Exception e) { - logger.error("{}", e); - return false; - } - } - // check - check(rocks); - } catch (Exception e) { - logger.error("{}", e); - return false; - } finally { - try { - level.close(); - rocks.close(); - JniDBFactory.popMemoryPool(); - } catch (Exception e1) { - logger.error("{}", e1); - } - } - return dstDbKeyCount == srcDbKeyCount && dstDbKeySum == srcDbKeySum - && dstDbValueSum == srcDbValueSum; - } - - private void check(RocksDB rocks) throws RocksDBException { - logger.info("check database {} start", this.dbName); - // manually call CompactRange() - logger.info("compact database {} start", this.dbName); - rocks.compactRange(); - logger.info("compact database {} end", this.dbName); - // check - try (org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); - RocksIterator rocksIterator = rocks.newIterator(r)) { - for (rocksIterator.seekToFirst(); rocksIterator.isValid(); rocksIterator.next()) { - byte[] key = rocksIterator.key(); - byte[] value = rocksIterator.value(); - dstDbKeyCount++; - dstDbKeySum = byteArrayToIntWithOne(dstDbKeySum, key); - dstDbValueSum = byteArrayToIntWithOne(dstDbValueSum, value); - } - } - logger.info("check database {} end", this.dbName); - } - - public boolean createEngine(String dir) { - String enginePath = dir + File.separator + "engine.properties"; - - if (!FileUtil.createFileIfNotExists(enginePath)) { - return false; - } - - return PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); - } - - public boolean checkDone(String dir) { - String enginePath = dir + File.separator + "engine.properties"; - return FileUtil.isExists(enginePath); - - } - - public boolean doConvert() throws Exception { - - if (checkDone(this.dstDbPath.toString())) { - logger.info(" {} is done, skip it.", this.dbName); - return true; - } - - File levelDbFile = srcDbPath.toFile(); - if (!levelDbFile.exists()) { - logger.info(" {} does not exist.", srcDbPath.toString()); - return false; - } - - DB level = newLevelDb(srcDbPath); - - if (this.dstDbPath.toFile().exists()) { - logger.info(" {} begin to clear exist database directory", this.dbName); - FileUtil.deleteDir(this.dstDbPath.toFile()); - logger.info(" {} clear exist database directory done.", this.dbName); - } - - FileUtil.createDirIfNotExists(dstDir); - RocksDB rocks = newRocksDb(dstDbPath); - - logger.info("Convert database {} start", this.dbName); - boolean result = convertLevelToRocksBatchIterator(level, rocks) - && createEngine(dstDbPath.toString()); - long etime = System.currentTimeMillis(); - - if (result) { - logger.info("Convert database {} successful end with {} key-value {} minutes", - this.dbName, this.srcDbKeyCount, (etime - this.startTime) / 1000.0 / 60); - } else { - logger.info("Convert database {} failure", this.dbName); - if (this.dstDbPath.toFile().exists()) { - logger.info(" {} begin to clear exist database directory", this.dbName); - FileUtil.deleteDir(this.dstDbPath.toFile()); - logger.info(" {} clear exist database directory done.", this.dbName); - } - } - return result; - } - - public long byteArrayToIntWithOne(long sum, byte[] b) { - for (byte oneByte : b) { - sum += oneByte; - } - return sum; - } -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 0fd87eb5de0..c0bd6aaaf49 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -50,7 +50,6 @@ public static void load(String path) { * Start the FullNode. */ public static void main(String[] args) { - logger.info("Full node running."); Args.setParam(args, Constant.TESTNET_CONF); CommonParameter parameter = Args.getInstance(); @@ -62,6 +61,16 @@ public static void main(String[] args) { Args.printHelp(jCommander); return; } + if (parameter.isKeystore()) { + KeystoreFactory.exec(); + return; + } + if (parameter.isSolidityNode()) { + SolidityNode.run(); + return; + } else { + logger.info("Full node running."); + } if (Args.getInstance().isDebug()) { logger.info("in debug mode, it won't check energy time"); diff --git a/framework/src/main/java/org/tron/program/KeystoreFactory.java b/framework/src/main/java/org/tron/program/KeystoreFactory.java index bfd2df22856..2df7c420fa7 100755 --- a/framework/src/main/java/org/tron/program/KeystoreFactory.java +++ b/framework/src/main/java/org/tron/program/KeystoreFactory.java @@ -1,6 +1,5 @@ package org.tron.program; -import com.beust.jcommander.JCommander; import java.io.File; import java.io.IOException; import java.util.Scanner; @@ -11,8 +10,6 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Utils; -import org.tron.core.Constant; -import org.tron.core.config.args.Args; import org.tron.core.exception.CipherException; import org.tron.keystore.Credentials; import org.tron.keystore.WalletUtils; @@ -22,15 +19,8 @@ public class KeystoreFactory { private static final String FilePath = "Wallet"; - public static void main(String[] args) { - Args.setParam(args, Constant.TESTNET_CONF); + public static void exec() { KeystoreFactory cli = new KeystoreFactory(); - - JCommander.newBuilder() - .addObject(cli) - .build() - .parse(args); - cli.run(); } diff --git a/framework/src/main/java/org/tron/program/SolidityNode.java b/framework/src/main/java/org/tron/program/SolidityNode.java index 4cf71177803..ca5456e3ba5 100644 --- a/framework/src/main/java/org/tron/program/SolidityNode.java +++ b/framework/src/main/java/org/tron/program/SolidityNode.java @@ -56,9 +56,8 @@ public SolidityNode(Manager dbManager) { /** * Start the SolidityNode. */ - public static void main(String[] args) { + public static void run() { logger.info("Solidity node is running."); - Args.setParam(args, Constant.TESTNET_CONF); CommonParameter parameter = CommonParameter.getInstance(); logger.info("index switch is {}", diff --git a/framework/src/test/java/org/tron/program/DBConvertTest.java b/framework/src/test/java/org/tron/program/DBConvertTest.java deleted file mode 100644 index 7b3f797d627..00000000000 --- a/framework/src/test/java/org/tron/program/DBConvertTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.tron.program; - -import static org.fusesource.leveldbjni.JniDBFactory.factory; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.UUID; -import org.iq80.leveldb.DB; -import org.iq80.leveldb.Options; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.FileUtil; -import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; -import org.tron.core.capsule.MarketOrderIdListCapsule; -import org.tron.core.capsule.utils.MarketUtils; - -public class DBConvertTest { - - - private static final String INPUT_DIRECTORY = "output-directory/convert-database/"; - private static final String OUTPUT_DIRECTORY = "output-directory/convert-database-dest/"; - private static final String ACCOUNT = "account"; - private static final String MARKET = "market_pair_price_to_order"; - - - @BeforeClass - public static void init() throws IOException { - if (new File(INPUT_DIRECTORY).mkdirs()) { - initDB(new File(INPUT_DIRECTORY,ACCOUNT)); - initDB(new File(INPUT_DIRECTORY,MARKET)); - } - } - - private static void initDB(File file) throws IOException { - Options dbOptions = DBConvert.newDefaultLevelDbOptions(); - if (file.getName().contains("market_pair_price_to_order")) { - dbOptions.comparator(new MarketOrderPriceComparatorForLevelDB()); - try (DB db = factory.open(file,dbOptions)) { - - byte[] sellTokenID1 = ByteArray.fromString("100"); - byte[] buyTokenID1 = ByteArray.fromString("200"); - byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2001L - ); - byte[] pairPriceKey2 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2002L - ); - byte[] pairPriceKey3 = MarketUtils.createPairPriceKey( - sellTokenID1, - buyTokenID1, - 1000L, - 2003L - ); - - MarketOrderIdListCapsule capsule1 = new MarketOrderIdListCapsule(ByteArray.fromLong(1), - ByteArray.fromLong(1)); - MarketOrderIdListCapsule capsule2 = new MarketOrderIdListCapsule(ByteArray.fromLong(2), - ByteArray.fromLong(2)); - MarketOrderIdListCapsule capsule3 = new MarketOrderIdListCapsule(ByteArray.fromLong(3), - ByteArray.fromLong(3)); - - //Use out-of-order insertion，key in store should be 1,2,3 - db.put(pairPriceKey1, capsule1.getData()); - db.put(pairPriceKey2, capsule2.getData()); - db.put(pairPriceKey3, capsule3.getData()); - } - - } else { - try (DB db = factory.open(file,dbOptions)) { - for (int i = 0; i < 100; i++) { - byte[] bytes = UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8); - db.put(bytes, bytes); - } - } - } - - } - - @AfterClass - public static void destroy() { - FileUtil.deleteDir(new File(INPUT_DIRECTORY)); - FileUtil.deleteDir(new File(OUTPUT_DIRECTORY)); - } - - @Test - public void testRun() { - String[] args = new String[] { INPUT_DIRECTORY, OUTPUT_DIRECTORY }; - Assert.assertEquals(0, DBConvert.run(args)); - } - - @Test - public void testNotExist() { - String[] args = new String[] {OUTPUT_DIRECTORY + File.separator + UUID.randomUUID(), - OUTPUT_DIRECTORY}; - Assert.assertEquals(404, DBConvert.run(args)); - } - - @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {file.toString(), OUTPUT_DIRECTORY}; - Assert.assertEquals(0, DBConvert.run(args)); - } -} diff --git a/framework/src/test/java/org/tron/program/SolidityNodeTest.java b/framework/src/test/java/org/tron/program/SolidityNodeTest.java index a77fa2fa8c6..ed6166cd764 100755 --- a/framework/src/test/java/org/tron/program/SolidityNodeTest.java +++ b/framework/src/test/java/org/tron/program/SolidityNodeTest.java @@ -1,7 +1,5 @@ package org.tron.program; -import java.io.File; -import java.io.IOException; import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.junit.AfterClass; @@ -26,12 +24,8 @@ public class SolidityNodeTest extends BaseTest { SolidityNodeHttpApiService solidityNodeHttpApiService; static { - try { - Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); - } catch (IOException e) { - Assert.fail("create temp directory failed."); - } - Args.getInstance().setSolidityNode(true); + Args.setParam(new String[]{"-d", dbPath(), "--solidity" + }, Constant.TEST_CONF); } /** @@ -41,26 +35,6 @@ public class SolidityNodeTest extends BaseTest { public static void init() { } - /** - * remo db when after test. - */ - @AfterClass - public static void removeDb() { - Args.clearParam(); - } - - private static Boolean deleteFolder(File index) { - if (!index.isDirectory() || index.listFiles().length <= 0) { - return index.delete(); - } - for (File file : index.listFiles()) { - if (null != file && !deleteFolder(file)) { - return false; - } - } - return index.delete(); - } - @Test public void testSolidityArgs() { Assert.assertNotNull(Args.getInstance().getTrustNodeAddr()); diff --git a/plugins/build.gradle b/plugins/build.gradle index b85a435dc30..8e018820da9 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -4,6 +4,7 @@ plugins { apply plugin: 'application' apply plugin: 'checkstyle' +apply plugin: 'com.github.johnrengelman.shadow' jacoco { toolVersion = "0.8.4" @@ -11,7 +12,7 @@ jacoco { def versions = [ checkstyle: '8.7', ] -mainClassName = 'org.tron.plugins.ArchiveManifest' +mainClassName = 'org.tron.plugins.Toolkit' group 'org.tron' version '1.0.0' @@ -133,23 +134,24 @@ def createScript(project, mainClass, name) { applicationDistribution.from("../gradle/java-tron.vmoptions") { into "bin" } -createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest') createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit') -def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' -def skipArchive = hasProperty('skipArchive') ? true : false -def skipAll = hasProperty('skipAll') ? true : false -if (releaseBinary == 'true') { - artifacts { - archives(binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit')) - } - if (!skipAll) { - if (!skipArchive) { - artifacts { - archives(binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest')) - } - } - } +shadowJar { + baseName = 'Toolkit' + classifier = null + version = null + // exclude these files for bouncycastle + exclude "META-INF/*.SF" + exclude "META-INF/*.DSA" + exclude "META-INF/*.RSA" + mergeServiceFiles() // https://github.com/grpc/grpc-java/issues/10853 +} +shadowDistZip { + enabled = false +} + +startShadowScripts { + enabled = false } task copyToParent(type: Copy) { diff --git a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java deleted file mode 100644 index 4d54df6d299..00000000000 --- a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java +++ /dev/null @@ -1,271 +0,0 @@ -package org.tron.plugins; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.iq80.leveldb.CompressionType; -import org.iq80.leveldb.DB; -import org.iq80.leveldb.Options; -import org.iq80.leveldb.impl.Filename; -import picocli.CommandLine; -import picocli.CommandLine.Option; - -@Slf4j(topic = "archive") -/* - a helper to rewrite leveldb manifest. - */ -public class ArchiveManifest implements Callable { - - - private static final String KEY_ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - - private final Path srcDbPath; - private final String name; - private final Options options; - private final long startTime; - - - private static final int CPUS = Runtime.getRuntime().availableProcessors(); - - public ArchiveManifest(String src, String name, int maxManifestSize, int maxBatchSize) { - this.name = name; - this.srcDbPath = Paths.get(src, name); - this.startTime = System.currentTimeMillis(); - this.options = newDefaultLevelDbOptions(); - this.options.maxManifestSize(maxManifestSize); - this.options.maxBatchSize(maxBatchSize); - } - - @Override - public Boolean call() throws Exception { - return doArchive(); - } - - public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { - org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); - dbOptions.createIfMissing(true); - dbOptions.paranoidChecks(true); - dbOptions.verifyChecksums(true); - dbOptions.compressionType(CompressionType.SNAPPY); - dbOptions.blockSize(4 * 1024); - dbOptions.writeBufferSize(10 * 1024 * 1024); - dbOptions.cacheSize(10 * 1024 * 1024L); - dbOptions.maxOpenFiles(1000); - dbOptions.maxBatchSize(64_000); - dbOptions.maxManifestSize(128); - return dbOptions; - } - - public static void main(String[] args) { - int code = run(args); - logger.info("exit code {}.", code); - System.out.printf("exit code %d.\n", code); - System.exit(code); - } - - public static int run(String[] args) { - Args parameters = new Args(); - CommandLine commandLine = new CommandLine(parameters); - commandLine.parseArgs(args); - if (parameters.help) { - commandLine.usage(System.out); - return 0; - } - - File dbDirectory = new File(parameters.databaseDirectory); - if (!dbDirectory.exists()) { - logger.info("Directory {} does not exist.", parameters.databaseDirectory); - return 404; - } - - if (!dbDirectory.isDirectory()) { - logger.info("{} is not directory.", parameters.databaseDirectory); - return 405; - } - - List files = Arrays.stream(Objects.requireNonNull(dbDirectory.listFiles())) - .filter(File::isDirectory).collect( - Collectors.toList()); - - if (files.isEmpty()) { - logger.info("Directory {} does not contain any database.", parameters.databaseDirectory); - return 0; - } - final long time = System.currentTimeMillis(); - final List> res = new ArrayList<>(); - final ThreadPoolExecutor executor = new ThreadPoolExecutor( - CPUS, 16 * CPUS, 1, TimeUnit.MINUTES, - new ArrayBlockingQueue<>(CPUS, true), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.CallerRunsPolicy()); - - executor.allowCoreThreadTimeOut(true); - - files.forEach(f -> res.add( - executor.submit(new ArchiveManifest(parameters.databaseDirectory, f.getName(), - parameters.maxManifestSize, parameters.maxBatchSize)))); - int fails = res.size(); - - for (Future re : res) { - try { - if (Boolean.TRUE.equals(re.get())) { - fails--; - } - } catch (InterruptedException e) { - logger.error("{}", e); - Thread.currentThread().interrupt(); - } catch (ExecutionException e) { - logger.error("{}", e); - } - } - - executor.shutdown(); - logger.info("DatabaseDirectory:{}, maxManifestSize:{}, maxBatchSize:{}," - + "database reopen use {} seconds total.", - parameters.databaseDirectory, parameters.maxManifestSize, parameters.maxBatchSize, - (System.currentTimeMillis() - time) / 1000); - if (fails > 0) { - logger.error("Failed!!!!!!!!!!!!!!!!!!!!!!!! size:{}", fails); - } - return fails; - } - - public void open() throws IOException { - DB database = factory.open(this.srcDbPath.toFile(), this.options); - database.close(); - } - - public boolean checkManifest(String dir) throws IOException { - // Read "CURRENT" file, which contains a pointer to the current manifest file - File currentFile = new File(dir, Filename.currentFileName()); - if (!currentFile.exists()) { - return false; - } - String currentName = com.google.common.io.Files.asCharSource(currentFile, UTF_8).read(); - if (currentName.isEmpty() || currentName.charAt(currentName.length() - 1) != '\n') { - return false; - } - currentName = currentName.substring(0, currentName.length() - 1); - File current = new File(dir, currentName); - if (!current.isFile()) { - return false; - } - long maxSize = options.maxManifestSize(); - if (maxSize < 0) { - return false; - } - logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { - logger.info("Db {} ignored.", this.name); - return false; - } - return current.length() >= maxSize * 1024 * 1024; - } - - public boolean doArchive() throws IOException { - File levelDbFile = srcDbPath.toFile(); - if (!levelDbFile.exists()) { - logger.info("File {},does not exist, ignored.", srcDbPath.toString()); - return true; - } - if (!checkManifest(levelDbFile.toString())) { - logger.info("Db {},no need, ignored.", levelDbFile.toString()); - return true; - } - if (!checkEngine()) { - logger.info("Db {},not leveldb, ignored.", this.name); - return true; - } - open(); - logger.info("Db {} archive use {} ms.", this.name, (System.currentTimeMillis() - startTime)); - return true; - } - - public boolean checkEngine() { - String dir = this.srcDbPath.toString(); - String enginePath = dir + File.separator + "engine.properties"; - if (!new File(enginePath).exists() && !writeProperty(enginePath, KEY_ENGINE, LEVELDB)) { - return false; - } - String engine = readProperty(enginePath, KEY_ENGINE); - return LEVELDB.equals(engine); - } - - public static String readProperty(String file, String key) { - try (FileInputStream fileInputStream = new FileInputStream(file); - InputStream inputStream = new BufferedInputStream(fileInputStream)) { - Properties prop = new Properties(); - prop.load(inputStream); - return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1), - UTF_8); - } catch (Exception e) { - logger.error("{}", e); - return ""; - } - } - - public static boolean writeProperty(String file, String key, String value) { - try (OutputStream out = new FileOutputStream(file); - FileInputStream fis = new FileInputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, UTF_8))) { - BufferedReader bf = new BufferedReader(new InputStreamReader(fis, UTF_8)); - Properties properties = new Properties(); - properties.load(bf); - properties.setProperty(key, value); - properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); - } catch (Exception e) { - logger.warn("{}", e); - return false; - } - return true; - } - - public static class Args { - - @Option(names = {"-d", "--database-directory"}, - defaultValue = "output-directory/database", - description = "java-tron database directory. Default: ${DEFAULT-VALUE}") - private String databaseDirectory; - - @Option(names = {"-b", "--batch-size" }, - defaultValue = "80000", - description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") - private int maxBatchSize; - - @Option(names = {"-m", "--manifest-size" }, - defaultValue = "0", - description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") - private int maxManifestSize; - - @Option(names = {"-h", "--help"}, help = true) - private boolean help; - } -} \ No newline at end of file diff --git a/plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java b/plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java deleted file mode 100644 index 4fd9e537d05..00000000000 --- a/plugins/src/test/java/org/tron/plugins/ArchiveManifestTest.java +++ /dev/null @@ -1,130 +0,0 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.util.Properties; -import java.util.UUID; - -import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -@Slf4j -public class ArchiveManifestTest { - - private static final String OUTPUT_DIRECTORY = "output-directory/database/archiveManifest"; - - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; - private static final String ACCOUNT = "account"; - private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; - - - - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); - - } - - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); - } - - @Test - public void testRun() { - String[] args = new String[] { "-d", OUTPUT_DIRECTORY }; - Assert.assertEquals(0, ArchiveManifest.run(args)); - } - - @Test - public void testHelp() { - String[] args = new String[] {"-h"}; - Assert.assertEquals(0, ArchiveManifest.run(args)); - } - - @Test - public void testMaxManifest() { - String[] args = new String[] {"-d", OUTPUT_DIRECTORY, "-m", "128"}; - Assert.assertEquals(0, ArchiveManifest.run(args)); - } - - @Test - public void testNotExist() { - String[] args = new String[] {"-d", OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()}; - Assert.assertEquals(404, ArchiveManifest.run(args)); - } - - @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"-d", file.toString()}; - Assert.assertEquals(0, ArchiveManifest.run(args)); - } - - private static void writeProperty(String filename, String key, String value) throws IOException { - File file = new File(filename); - if (!file.exists()) { - file.createNewFile(); - } - - try (FileInputStream fis = new FileInputStream(file); - OutputStream out = new FileOutputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, - StandardCharsets.UTF_8))) { - BufferedReader bf = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); - Properties properties = new Properties(); - properties.load(bf); - properties.setProperty(key, value); - properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); - } catch (Exception e) { - logger.warn("{}", e); - } - } - - /** - * delete directory. - */ - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - String[] children = dir.list(); - assert children != null; - for (String child : children) { - boolean success = deleteDir(new File(dir, child)); - if (!success) { - logger.warn("can't delete dir:" + dir); - return false; - } - } - } - return dir.delete(); - } -} diff --git a/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java b/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java index 10bed418764..2066464a8ad 100644 --- a/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.tron.plugins.utils.DBUtils; import picocli.CommandLine; @Slf4j @@ -38,15 +39,15 @@ public class DbArchiveTest { @BeforeClass public static void init() throws IOException { File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file, DBUtils.newDefaultLevelDbOptions()).close(); writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file,DBUtils.newDefaultLevelDbOptions()).close(); writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file,DBUtils.newDefaultLevelDbOptions()).close(); writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); } diff --git a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java index c1bc6b470fc..c51cc2b3806 100644 --- a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java @@ -12,6 +12,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @@ -32,15 +33,15 @@ public class DbMoveTest { @BeforeClass public static void init() throws IOException { File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file, DBUtils.newDefaultLevelDbOptions()).close(); FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file, DBUtils.newDefaultLevelDbOptions()).close(); FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); + factory.open(file, DBUtils.newDefaultLevelDbOptions()).close(); FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); } diff --git a/start.sh b/start.sh index 89f13cf25a7..7b97c839b91 100644 --- a/start.sh +++ b/start.sh @@ -377,12 +377,12 @@ rebuildManifest() { return fi - ARCHIVE_JAR='ArchiveManifest.jar' + ARCHIVE_JAR='Toolkit.jar' if [[ -f $ARCHIVE_JAR ]]; then echo 'info: execute rebuild manifest.' $JAVACMD -jar $ARCHIVE_JAR -d $REBUILD_DIR -m $REBUILD_MANIFEST_SIZE -b $REBUILD_BATCH_SIZE else - echo 'info: download the rebuild manifest plugin from the github' + echo 'info: download Toolkit from the github' local latest=$(`echo getLatestReleaseVersion`) download $RELEASE_URL/download/GreatVoyage-v"$latest"/$ARCHIVE_JAR $ARCHIVE_JAR if [[ $download == 0 ]]; then