From 3c5e8576ae5646d8130180a3b5e8b423d1ddf4fd Mon Sep 17 00:00:00 2001 From: apachemycat Date: Mon, 16 Jan 2017 10:58:53 +0800 Subject: [PATCH 1/7] candidate-commiter/readme.md --- candidate-commiter/readme.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 candidate-commiter/readme.md diff --git a/candidate-commiter/readme.md b/candidate-commiter/readme.md new file mode 100644 index 0000000..e69de29 From 8f9555eeb489d2d9d25180bd575564b6f6ce59b4 Mon Sep 17 00:00:00 2001 From: "547557645@qq.com" <547557645@qq.com> Date: Sun, 22 Jan 2017 09:59:35 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=B0=81=E8=A3=85netty=20pooled=20Direct?= =?UTF-8?q?=20Memory=20=E6=8E=A5=E5=8F=A3=EF=BC=8C=E4=B8=BAmycat=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E5=86=85=E5=AD=98=E6=B1=A0=E7=AE=A1=E7=90=86=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mycat-Core/pom.xml | 6 + .../mycat/memalloc/MyCatMemoryAllocator.java | 197 ++++++++++++++++++ .../memalloc/TestMycatMemoryAlloctor.java | 53 +++++ 3 files changed, 256 insertions(+) create mode 100644 Mycat-Core/src/main/java/io/mycat/memalloc/MyCatMemoryAllocator.java create mode 100644 Mycat-Core/src/test/java/io/mycat/memalloc/TestMycatMemoryAlloctor.java diff --git a/Mycat-Core/pom.xml b/Mycat-Core/pom.xml index 32ef931..e740350 100644 --- a/Mycat-Core/pom.xml +++ b/Mycat-Core/pom.xml @@ -72,6 +72,12 @@ 2.3 + + + io.netty + netty-buffer + 4.1.7.Final + diff --git a/Mycat-Core/src/main/java/io/mycat/memalloc/MyCatMemoryAllocator.java b/Mycat-Core/src/main/java/io/mycat/memalloc/MyCatMemoryAllocator.java new file mode 100644 index 0000000..2f390f2 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/memalloc/MyCatMemoryAllocator.java @@ -0,0 +1,197 @@ +package io.mycat.memalloc; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.util.internal.PlatformDependent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 封装netty pooled Direct Memory 接口，为mycat提供内存池管理功能 + * + * @author zagnix + * @create 2017-01-18 11:01 + */ + +public class MyCatMemoryAllocator implements ByteBufAllocator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MyCatMemoryAllocator.class); + + + private final static MyCatMemoryAllocator INSTANCE = + new MyCatMemoryAllocator(Runtime.getRuntime().availableProcessors()*2); + + /** netty memory pool alloctor*/ + private final PooledByteBufAllocator alloc; + /**arena 的数量，一般设置cpu cores*2 */ + private final int numberOfArenas; + + /** ChunkSize 大小 = pageSize << maxOrder */ + private final int chunkSize; + + /** + * numberOfArenas 设置为处理器cores*2 + * @param numberOfArenas + */ + public MyCatMemoryAllocator(int numberOfArenas){ + this.numberOfArenas = numberOfArenas; + if (!PlatformDependent.hasUnsafe()) { + LOGGER.warn("Using direct memory, but sun.misc.Unsafe not available."); + } + boolean preferDirect = true; + + int pageSize = 8192; + int maxOrder = 11; + + this.chunkSize = pageSize << maxOrder; + + int numDirectArenas = numberOfArenas; + + int numHeapArenas = 0; + + this.alloc = new PooledByteBufAllocator( + preferDirect, + numHeapArenas, + numDirectArenas, + pageSize, + maxOrder, + 512/**tinycache*/, + 256/**smallcache*/, + 64/**normalcache*/, + true/**使用Thread Local Cache*/); + } + + + public static MyCatMemoryAllocator getINSTANCE() { + return INSTANCE; + } + + /** + * + * @return alloc + */ + public PooledByteBufAllocator getAlloc() { + return alloc; + } + + /** + * Returns the number of arenas. + * + * @return Number of arenas. + */ + int getNumberOfArenas() { + return numberOfArenas; + } + + /** + * Returns the chunk size. + * + * @return Chunk size. + */ + int getChunkSize() { + return chunkSize; + } + + + @Override + public ByteBuf buffer() { + return alloc.buffer(); + } + + @Override + public ByteBuf buffer(int initialCapacity) { + return alloc.buffer(initialCapacity); + } + + @Override + public ByteBuf buffer(int initialCapacity, int maxCapacity) { + return alloc.buffer(initialCapacity, maxCapacity); + } + + @Override + public ByteBuf ioBuffer() { + return alloc.ioBuffer(); + } + + @Override + public ByteBuf ioBuffer(int initialCapacity) { + return alloc.ioBuffer(initialCapacity); + } + + @Override + public ByteBuf ioBuffer(int initialCapacity, int maxCapacity) { + return alloc.ioBuffer(initialCapacity, maxCapacity); + } + + @Override + public ByteBuf heapBuffer() { + throw new UnsupportedOperationException("Heap buffer"); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity) { + throw new UnsupportedOperationException("Heap buffer"); + } + + @Override + public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) { + throw new UnsupportedOperationException("Heap buffer"); + } + + @Override + public ByteBuf directBuffer() { + return alloc.directBuffer(); + } + + @Override + public ByteBuf directBuffer(int initialCapacity) { + return alloc.directBuffer(initialCapacity); + } + + @Override + public ByteBuf directBuffer(int initialCapacity, int maxCapacity) { + return alloc.directBuffer(initialCapacity, maxCapacity); + } + + @Override + public CompositeByteBuf compositeBuffer() { + return alloc.compositeBuffer(); + } + + @Override + public CompositeByteBuf compositeBuffer(int maxNumComponents) { + return alloc.compositeBuffer(maxNumComponents); + } + + @Override + public CompositeByteBuf compositeHeapBuffer() { + throw new UnsupportedOperationException("Heap buffer"); + } + + @Override + public CompositeByteBuf compositeHeapBuffer(int maxNumComponents) { + throw new UnsupportedOperationException("Heap buffer"); + } + + @Override + public CompositeByteBuf compositeDirectBuffer() { + return alloc.compositeDirectBuffer(); + } + + @Override + public CompositeByteBuf compositeDirectBuffer(int maxNumComponents) { + return alloc.compositeDirectBuffer(maxNumComponents); + } + + @Override + public boolean isDirectBufferPooled() { + return alloc.isDirectBufferPooled(); + } + + @Override + public int calculateNewCapacity(int i, int i1) { + return 0; + } +} diff --git a/Mycat-Core/src/test/java/io/mycat/memalloc/TestMycatMemoryAlloctor.java b/Mycat-Core/src/test/java/io/mycat/memalloc/TestMycatMemoryAlloctor.java new file mode 100644 index 0000000..acc7270 --- /dev/null +++ b/Mycat-Core/src/test/java/io/mycat/memalloc/TestMycatMemoryAlloctor.java @@ -0,0 +1,53 @@ +package io.mycat.memalloc; + +import io.netty.buffer.ByteBuf; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; + +/** + * @author zagnix + * @create 2017-01-18 11:19 + */ + +public class TestMycatMemoryAlloctor { + @Test + public void testMemAlloc(){ + final MyCatMemoryAllocator memoryAllocator = + new MyCatMemoryAllocator(Runtime.getRuntime().availableProcessors()*2); + + for (int i = 0; i <100 ; i++) { + ByteBuf byteBuf = memoryAllocator.directBuffer(128); + byteBuf.writeBytes("helll world".getBytes()); + ByteBuffer byteBuffer = byteBuf.nioBuffer(0,128); + String srt = getString(byteBuffer); + byteBuf.release(); + System.out.println("refCnt:" + byteBuf.refCnt()); + // byteBuf.writeBytes(byteBuffer); + // System.out.println("refCnt:" + byteBuf.refCnt()); + } + } + + public static String getString(ByteBuffer buffer) { + Charset charset = null; + CharsetDecoder decoder = null; + CharBuffer charBuffer = null; + try { + charset = Charset.forName("UTF-8"); + decoder = charset.newDecoder(); + charBuffer = decoder.decode(buffer.asReadOnlyBuffer()); + return charBuffer.toString(); + } catch (Exception ex) { + ex.printStackTrace(); + return "error"; + } + } + + public static ByteBuffer getByteBuffer(String str) + { + return ByteBuffer.wrap(str.getBytes()); + } +} From 0f5f9173f181712e1392812318eb53a4be37e493 Mon Sep 17 00:00:00 2001 From: "547557645@qq.com" <547557645@qq.com> Date: Sun, 22 Jan 2017 10:38:16 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=B8=A6=E6=B3=A8?= =?UTF-8?q?=E8=A7=A3select=E8=AF=AD=E5=8F=A5/*!mycat:cacheable=3Dtrue,kv?= =?UTF-8?q?=E5=B1=9E=E6=80=A7/sql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../MySQLBackendConnectionHandler.java | 8 +- .../backend/callback/LoginRespCallback.java | 2 +- .../callback/SQLResCacheHintHandler.java | 79 ++++++++++ .../impl/AbstractSchemaSQLCommandHandler.java | 28 +++- .../front/MySQLFrontConnectionHandler.java | 6 +- .../main/java/io/mycat/mysql/BufferUtil.java | 147 ++++++++++++++++++ .../io/mycat/mysql/packet/CommandPacket.java | 126 +++++++++++++++ .../io/mycat/net2/ByteBufConDataBuffer.java | 143 +++++++++++++++++ .../main/java/io/mycat/net2/Connection.java | 31 +++- .../java/io/mycat/sqlcache/HintHandler.java | 12 ++ .../java/io/mycat/sqlcache/HintSQLInfo.java | 88 +++++++++++ .../java/io/mycat/sqlcache/HintSQLParser.java | 79 ++++++++++ .../sqlcache/SQLResultsCacheService.java | 131 ++++++++++++++++ 13 files changed, 865 insertions(+), 15 deletions(-) create mode 100644 Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java create mode 100644 Mycat-Core/src/main/java/io/mycat/mysql/BufferUtil.java create mode 100644 Mycat-Core/src/main/java/io/mycat/mysql/packet/CommandPacket.java create mode 100644 Mycat-Core/src/main/java/io/mycat/net2/ByteBufConDataBuffer.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/HintHandler.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLInfo.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java diff --git a/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java b/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java index db91d12..001c610 100644 --- a/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java @@ -60,6 +60,7 @@ public void handleReadEvent(MySQLBackendConnection con) throws IOException{ { return; } + length = MySQLConnection.getPacketLength(dataBuffer, offset); if(length+offset>limit) { @@ -70,11 +71,12 @@ public void handleReadEvent(MySQLBackendConnection con) throws IOException{ byte packetType = dataBuffer.getByte(offset+MySQLConnection.msyql_packetHeaderSize); int pkgStartPos=offset; - offset += length; - dataBuffer.setReadingPos(offset); - + LOGGER.info("received pkg ,length "+length+" type "+packetType+" cur total length "+limit); con.getUserCallback().handleResponse(con, dataBuffer, packetType, pkgStartPos, length); + + offset += length; + dataBuffer.setReadingPos(offset); } diff --git a/Mycat-Core/src/main/java/io/mycat/backend/callback/LoginRespCallback.java b/Mycat-Core/src/main/java/io/mycat/backend/callback/LoginRespCallback.java index beace7d..be063a7 100644 --- a/Mycat-Core/src/main/java/io/mycat/backend/callback/LoginRespCallback.java +++ b/Mycat-Core/src/main/java/io/mycat/backend/callback/LoginRespCallback.java @@ -91,7 +91,7 @@ public void handleResponse(MySQLBackendConnection source, ConDataBuffer dataBuff packet = source.getHandshake(); if (packet == null) { byteBuff=dataBuffer.getBytes(pkgStartPos, pkgLen); - processHandShakePacket(source, byteBuff); + processHandShakePacket(source, byteBuff); // 发送认证数据包 source.authenticate(); break; diff --git a/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java new file mode 100644 index 0000000..763e763 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java @@ -0,0 +1,79 @@ +package io.mycat.backend.callback; + +import io.mycat.backend.BackConnectionCallback; +import io.mycat.backend.MySQLBackendConnection; +import io.mycat.front.MySQLFrontConnection; +import io.mycat.net2.ConDataBuffer; +import io.mycat.net2.Connection; +import io.mycat.net2.ConnectionException; +import io.mycat.sqlcache.HintHandler; +import io.mycat.sqlcache.HintSQLInfo; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * SQL 结果集缓存 HintHandler + * + * @author zagnix + * @create 2017-01-17 14:11 + */ + +public class SQLResCacheHintHandler implements BackConnectionCallback,HintHandler { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SQLResCacheHintHandler.class); + private int filedCount = 0; + private HintSQLInfo hintSQLInfo; + public SQLResCacheHintHandler(){ + this.hintSQLInfo = hintSQLInfo; + } + + @Override + public boolean handle(String sql) { + return false; + } + + @Override + public void connectionError(ConnectionException e, MySQLBackendConnection conn) { + LOGGER.error("Hint SQL connectionError"); + + } + + @Override + public void connectionAcquired(MySQLBackendConnection conn) { + LOGGER.error("Hint SQL connectionAcquired"); + ((MySQLFrontConnection)conn.getAttachement()).executePendingTask(); + } + + @Override + public void handleResponse(MySQLBackendConnection conn, ConDataBuffer dataBuffer, byte packetType, int pkgStartPos, int pkgLen) throws IOException { + //LOGGER.error("Hint SQL handleResponse"); + MySQLFrontConnection frontCon= (MySQLFrontConnection) conn.getAttachement(); + /** + * 直接透传给前端client即可， + * 缓存框架需要在这里将收到的数据写到缓存中 + */ + ByteBuffer packetBuffer = dataBuffer.getBytes(pkgStartPos,pkgLen); + frontCon.getWriteDataBuffer().putBytes(packetBuffer); + frontCon.enableWrite(false); + conn.setNextStatus(packetType); + + if (conn.getState()== Connection.STATE_IDLE) + { + LOGGER.debug("realese bakend connedtion "+conn); + conn.release(); + } + } + + @Override + public void connectionClose(MySQLBackendConnection conn, String reason) { + LOGGER.error("Hint SQL connectionClose"); + + } + + @Override + public void handlerError(Exception e, MySQLBackendConnection conn) { + LOGGER.error("Hint SQL handlerError"); + + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java b/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java index 0fad49b..47302f7 100644 --- a/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java @@ -27,6 +27,9 @@ import java.nio.ByteBuffer; import java.util.ArrayList; +import io.mycat.sqlcache.HintSQLInfo; +import io.mycat.sqlcache.HintSQLParser; +import io.mycat.sqlcache.SQLResultsCacheService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,9 +104,28 @@ private void doSQLCommand(MySQLFrontConnection frontCon, ConDataBuffer dataBuffe ByteBuffer byteBuff = dataBuffer.getBytes(pkgStartPos, pkgLen); MySQLMessage mm = new MySQLMessage(byteBuff); mm.position(5); - String sql = null; - sql = mm.readString(frontCon.getCharset()); - LOGGER.debug("received sql "+sql); + String sql = mm.readString(frontCon.getCharset()); + /** + * parser hit sql + * 需要改写sql语句，去掉前面的注释即可 + * 首先判断是否 + */ + HintSQLInfo hintSQL = HintSQLParser.parserHintSQL(sql); + if (hintSQL != null && hintSQL.isCache()){ + /** + * 0.判断是select语句 + * 1.改写sql语句，去掉前面的注释 + * 2.发送sql语句到后端执行。做好backend connection的Handler设置工作 + * 3.解析sql的结果集，决定是否异步缓存，一部分直接发送给您前端。 + * 4.如果当前sql的结果集已经缓存了。直接从本地缓存中拉去结果集即可 + */ + + if (!SQLResultsCacheService.getInstance().processHintSQL(frontCon,hintSQL,mm)){ + frontCon.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "cache no implementation"); + } + + return; + } // 执行查询 SQLInfo sqlInf=new SQLInfo(); int rs = ServerParse.parse(sql,sqlInf); diff --git a/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java b/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java index 6085f79..badf4bd 100644 --- a/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java @@ -88,8 +88,7 @@ public void handleReadEvent(final MySQLFrontConnection cnxn) throws IOException{ // 解析报文类型 final byte packetType = buffer.getByte(offset + MySQLConnection.msyql_packetHeaderSize); final int pkgStartPos = offset; - offset += length; - buffer.setReadingPos(offset); + // trace-protocol-packet // @author little-pan // @since 2016-09-29 @@ -100,6 +99,9 @@ public void handleReadEvent(final MySQLFrontConnection cnxn) throws IOException{ cnxn.getId(), buffer.hashCode(), pkgStartPos, length, packetType, limit, hexs); } cnxn.getSession().getCurCmdHandler().processCmd(cnxn, buffer, packetType, pkgStartPos, length); + + offset += length; + buffer.setReadingPos(offset); } } diff --git a/Mycat-Core/src/main/java/io/mycat/mysql/BufferUtil.java b/Mycat-Core/src/main/java/io/mycat/mysql/BufferUtil.java new file mode 100644 index 0000000..3f4ceb8 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/mysql/BufferUtil.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2013, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software;Designed and Developed mainly by many Chinese + * opensource volunteers. you can redistribute it and/or modify it under the + * terms of the GNU General Public License version 2 only, as published by the + * Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Any questions about this component can be directed to it's project Web address + * https://code.google.com/p/opencloudb/. + * + */ +package io.mycat.mysql; + +import java.nio.ByteBuffer; + +/** + * @author mycat + */ +public class BufferUtil { + + public static final void writeUB2(ByteBuffer buffer, int i) { + buffer.put((byte) (i & 0xff)); + buffer.put((byte) (i >>> 8)); + } + + public static final void writeUB3(ByteBuffer buffer, int i) { + buffer.put((byte) (i & 0xff)); + buffer.put((byte) (i >>> 8)); + buffer.put((byte) (i >>> 16)); + } + + public static final void writeInt(ByteBuffer buffer, int i) { + buffer.put((byte) (i & 0xff)); + buffer.put((byte) (i >>> 8)); + buffer.put((byte) (i >>> 16)); + buffer.put((byte) (i >>> 24)); + } + + public static final void writeFloat(ByteBuffer buffer, float f) { + writeInt(buffer, Float.floatToIntBits(f)); + } + + public static final void writeUB4(ByteBuffer buffer, long l) { + buffer.put((byte) (l & 0xff)); + buffer.put((byte) (l >>> 8)); + buffer.put((byte) (l >>> 16)); + buffer.put((byte) (l >>> 24)); + } + + public static final void writeLong(ByteBuffer buffer, long l) { + buffer.put((byte) (l & 0xff)); + buffer.put((byte) (l >>> 8)); + buffer.put((byte) (l >>> 16)); + buffer.put((byte) (l >>> 24)); + buffer.put((byte) (l >>> 32)); + buffer.put((byte) (l >>> 40)); + buffer.put((byte) (l >>> 48)); + buffer.put((byte) (l >>> 56)); + } + + public static final void writeDouble(ByteBuffer buffer, double d) { + writeLong(buffer, Double.doubleToLongBits(d)); + } + + public static final void writeLength(ByteBuffer buffer, long l) { + if (l < 251) { + buffer.put((byte) l); + } else if (l < 0x10000L) { + buffer.put((byte) 252); + writeUB2(buffer, (int) l); + } else if (l < 0x1000000L) { + buffer.put((byte) 253); + writeUB3(buffer, (int) l); + } else { + buffer.put((byte) 254); + writeLong(buffer, l); + } + } + + public static final void writeWithNull(ByteBuffer buffer, byte[] src) { + buffer.put(src); + buffer.put((byte) 0); + } + + public static final void writeWithLength(ByteBuffer buffer, byte[] src) { + int length = src.length; + if (length < 251) { + buffer.put((byte) length); + } else if (length < 0x10000L) { + buffer.put((byte) 252); + writeUB2(buffer, length); + } else if (length < 0x1000000L) { + buffer.put((byte) 253); + writeUB3(buffer, length); + } else { + buffer.put((byte) 254); + writeLong(buffer, length); + } + buffer.put(src); + } + + public static final void writeWithLength(ByteBuffer buffer, byte[] src, byte nullValue) { + if (src == null) { + buffer.put(nullValue); + } else { + writeWithLength(buffer, src); + } + } + + public static final int getLength(long length) { + if (length < 251) { + return 1; + } else if (length < 0x10000L) { + return 3; + } else if (length < 0x1000000L) { + return 4; + } else { + return 9; + } + } + + public static final int getLength(byte[] src) { + int length = src.length; + if (length < 251) { + return 1 + length; + } else if (length < 0x10000L) { + return 3 + length; + } else if (length < 0x1000000L) { + return 4 + length; + } else { + return 9 + length; + } + } + +} \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/mysql/packet/CommandPacket.java b/Mycat-Core/src/main/java/io/mycat/mysql/packet/CommandPacket.java new file mode 100644 index 0000000..4536b64 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/mysql/packet/CommandPacket.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2013, OpenCloudDB/MyCAT and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software;Designed and Developed mainly by many Chinese + * opensource volunteers. you can redistribute it and/or modify it under the + * terms of the GNU General Public License version 2 only, as published by the + * Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Any questions about this component can be directed to it's project Web address + * https://code.google.com/p/opencloudb/. + * + */ +package io.mycat.mysql.packet; + + +import io.mycat.mysql.BufferUtil; +import io.mycat.mysql.MySQLConnection; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +/** + * From client to server whenever the client wants the server to do something. + * + * 
+ * Bytes         Name
+ * -----         ----
+ * 1             command
+ * n             arg
+ * 
+ * command:      The most common value is 03 COM_QUERY, because
+ *               INSERT UPDATE DELETE SELECT etc. have this code.
+ *               The possible values at time of writing (taken
+ *               from /include/mysql_com.h for enum_server_command) are:
+ * 
+ *               #      Name                Associated client function
+ *               -      ----                --------------------------
+ *               0x00   COM_SLEEP           (none, this is an internal thread state)
+ *               0x01   COM_QUIT            mysql_close
+ *               0x02   COM_INIT_DB         mysql_select_db 
+ *               0x03   COM_QUERY           mysql_real_query
+ *               0x04   COM_FIELD_LIST      mysql_list_fields
+ *               0x05   COM_CREATE_DB       mysql_create_db (deprecated)
+ *               0x06   COM_DROP_DB         mysql_drop_db (deprecated)
+ *               0x07   COM_REFRESH         mysql_refresh
+ *               0x08   COM_SHUTDOWN        mysql_shutdown
+ *               0x09   COM_STATISTICS      mysql_stat
+ *               0x0a   COM_PROCESS_INFO    mysql_list_processes
+ *               0x0b   COM_CONNECT         (none, this is an internal thread state)
+ *               0x0c   COM_PROCESS_KILL    mysql_kill
+ *               0x0d   COM_DEBUG           mysql_dump_debug_info
+ *               0x0e   COM_PING            mysql_ping
+ *               0x0f   COM_TIME            (none, this is an internal thread state)
+ *               0x10   COM_DELAYED_INSERT  (none, this is an internal thread state)
+ *               0x11   COM_CHANGE_USER     mysql_change_user
+ *               0x12   COM_BINLOG_DUMP     sent by the slave IO thread to request a binlog
+ *               0x13   COM_TABLE_DUMP      LOAD TABLE ... FROM MASTER (deprecated)
+ *               0x14   COM_CONNECT_OUT     (none, this is an internal thread state)
+ *               0x15   COM_REGISTER_SLAVE  sent by the slave to register with the master (optional)
+ *               0x16   COM_STMT_PREPARE    mysql_stmt_prepare
+ *               0x17   COM_STMT_EXECUTE    mysql_stmt_execute
+ *               0x18   COM_STMT_SEND_LONG_DATA mysql_stmt_send_long_data
+ *               0x19   COM_STMT_CLOSE      mysql_stmt_close
+ *               0x1a   COM_STMT_RESET      mysql_stmt_reset
+ *               0x1b   COM_SET_OPTION      mysql_set_server_option
+ *               0x1c   COM_STMT_FETCH      mysql_stmt_fetch
+ * 
+ * arg:          The text of the command is just the way the user typed it, there is no processing
+ *               by the client (except removal of the final ';').
+ *               This field is not a null-terminated string; however,
+ *               the size can be calculated from the packet size,
+ *               and the MySQL client appends '\0' when receiving.
+ *               
+ * @see http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Command_Packet_.28Overview.29
+ *
+ * + * @author mycat + */ +public class CommandPacket extends MySQLPacket { + + public byte command; + public byte[] arg; + + public void read(ByteBuffer data) { + MySQLMessage mm = new MySQLMessage(data); + packetLength = mm.readUB3(); + packetId = mm.read(); + command = mm.read(); + arg = mm.readBytes(); + } + + public ByteBuffer write(MySQLConnection c) throws IOException { + ByteBuffer buffer = ByteBuffer.allocateDirect(256); + BufferUtil.writeUB3(buffer, calcPacketSize()); + buffer.put(packetId); + buffer.put(command); + buffer.put(arg); + buffer.flip(); + return buffer; + // c.getChannel().write(buffer); + } + + @Override + public int calcPacketSize() { + return 1 + arg.length; + } + + @Override + protected String getPacketInfo() { + return "MySQL Command Packet"; + } + + +} \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/net2/ByteBufConDataBuffer.java b/Mycat-Core/src/main/java/io/mycat/net2/ByteBufConDataBuffer.java new file mode 100644 index 0000000..3051e88 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/net2/ByteBufConDataBuffer.java @@ -0,0 +1,143 @@ +package io.mycat.net2; + +import io.mycat.memalloc.MyCatMemoryAllocator; +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; + +/** + * @author zagnix + * @create 2017-01-19 11:41 + */ + +public class ByteBufConDataBuffer implements ConDataBuffer { + + private ByteBuf buffer = null; + private final int initCapacity; + private final int maxCapacity; + + public ByteBufConDataBuffer(final int initCapacity,final int maxCapacity){ + this.initCapacity = initCapacity; + this.maxCapacity = maxCapacity; + this.buffer = MyCatMemoryAllocator.getINSTANCE().directBuffer(initCapacity); + } + + @Override + public int transferFrom(SocketChannel socketChanel) throws IOException { + /* int size=-1,len = 0; + int pkglen =0; + + while ((size=buffer.writeBytes(socketChanel,512))>0){ + len +=size; + } + */ + //System.out.println("transferFrom1 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + System.out.println("writable bytes :" + buffer.writableBytes()); + + int len = buffer.writeBytes(socketChanel,buffer.writableBytes()); + //System.out.println("transferFrom2 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + return len; + + } + + @Override + public void putBytes(ByteBuffer buf) throws IOException { + buffer.writeBytes(buf); + } + + @Override + public void putBytes(byte[] buf) throws IOException { + buffer.writeBytes(buf); + } + + @Override + public ByteBuffer beginWrite(int length) throws IOException { + System.out.println("beginWrite1 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + ByteBuffer byteBuffer =ByteBuffer.allocateDirect(length); //buffer.internalNioBuffer(buffer.writerIndex(),length); + System.out.println("beginWrite2 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + return byteBuffer; + } + + @Override + public void endWrite(ByteBuffer src) throws IOException { + buffer.writeBytes(src); + } + + @Override + public byte getByte(int index) throws IOException { + return buffer.getByte(index); + } + + @Override + public ByteBuffer getBytes(int index, int length) throws IOException { + buffer.readerIndex(index); + buffer.writerIndex(index+length); + ByteBuffer byteBuffer = buffer.nioBuffer(0,length); + buffer.readBytes(byteBuffer); + return byteBuffer; + } + + @Override + public int transferTo(SocketChannel socketChanel) throws IOException { + // System.out.println("transferTo1 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + int len = buffer.readBytes(socketChanel,buffer.readableBytes()); + //System.out.println("transferTo2 read index = " + buffer.readerIndex() + "write index = " + buffer.writerIndex()); + return len; + + } + + @Override + public int writingPos() throws IOException { + return buffer.writerIndex(); + } + + @Override + public int readPos() { + return buffer.readerIndex(); + } + + @Override + public int totalSize() { + return maxCapacity; + } + + @Override + public void setWritingPos(int writingPos) throws IOException { + buffer.setIndex(buffer.readerIndex(),writingPos); + } + + @Override + public void setReadingPos(int readingPos) { + buffer.setIndex(readingPos,buffer.writerIndex()); + } + + @Override + public boolean isFull() throws IOException { + return !buffer.isWritable(); + } + + @Override + public void recycle() { + //buffer.release(); + } + + public ByteBuf getBuffer() { + return buffer; + } + + public void setBuffer(ByteBuf buffer) { + this.buffer = buffer; + } + + public int getInitCapacity() { + return initCapacity; + } + + + public int getMaxCapacity() { + return maxCapacity; + } + +} diff --git a/Mycat-Core/src/main/java/io/mycat/net2/Connection.java b/Mycat-Core/src/main/java/io/mycat/net2/Connection.java index f610489..a8e36dc 100644 --- a/Mycat-Core/src/main/java/io/mycat/net2/Connection.java +++ b/Mycat-Core/src/main/java/io/mycat/net2/Connection.java @@ -246,13 +246,21 @@ protected void cleanup() { // 清理资源占用 if(readDataBuffer!=null) { - readDataBuffer.recycle(); - readDataBuffer=null; + try { + readDataBuffer.recycle(); + } catch (Exception e) { + e.printStackTrace(); + } + readDataBuffer=null; } if(this.writeDataBuffer!=null) { - writeDataBuffer.recycle(); - writeDataBuffer=null; + try { + writeDataBuffer.recycle(); + } catch (Exception e) { + e.printStackTrace(); + } + writeDataBuffer=null; } @@ -275,8 +283,19 @@ public void register(Selector selector, ReactorBufferPool myBufferPool) throws I String maprFileName=id+".rtmp"; String mapwFileName=id+".wtmp"; LOGGER.info("connection bytebuffer mapped "+maprFileName); - this.readDataBuffer =new MappedFileConDataBuffer(maprFileName); // 2 ,3 - writeDataBuffer=new MappedFileConDataBuffer3(mapwFileName); + + //TODO + /**使用MyCatMemoryAllocator分配Direct Buffer,再进行SocketChannel通信时候， + * NIO读写都会减少一次数据的拷贝,而使用FileChanel与SocketChannel数据交换时 + * 底层最终还是生成一个临时的Direct Buffer，用临时Direct Buffer写入或者读SocketChannel中 + * 后面考虑会使用netty中ByteBuf中的DirectBuffer进行网络IO通信。效率更高 + * */ + this.readDataBuffer =new MappedFileConDataBuffer3(maprFileName); // 2 ,3 + this.writeDataBuffer=new MappedFileConDataBuffer3(mapwFileName); + //存在bug暂不启用，以后统一是ByteBuf作为buffer进行NIO网络通信。 + // this.readDataBuffer = new ByteBufConDataBuffer(4096,16*1024*1024); + // this.writeDataBuffer = new ByteBufConDataBuffer(4096,16*1024*1024); + //新的client进来后，处理Server发送Client的handshake init packet this.handler.onConnected(this); } diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintHandler.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintHandler.java new file mode 100644 index 0000000..6f1b1f8 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintHandler.java @@ -0,0 +1,12 @@ +package io.mycat.sqlcache; + +/** + * Hint处理Handler + * + * @author zagnix + * @create 2017-01-17 14:07 + */ + +public interface HintHandler { + public boolean handle(String sql); +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLInfo.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLInfo.java new file mode 100644 index 0000000..f88dbe5 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLInfo.java @@ -0,0 +1,88 @@ +package io.mycat.sqlcache; + +import java.util.HashMap; +import java.util.Map; + +/** + * Hint SQL 携带的属性KV 和 真实执行的sql + * + * @author zagnix + * @version 1.0 + * @create 2017-01-16 17:32 + */ + +public class HintSQLInfo { + private String functionName; + private boolean isCache = false; + private HintHandler hintHandler; + private String hintSQL; + private String execSQL; + + /** + * paramsKv 中的Key必须是 cache-time,auto-refresh,access-count + * cache-time=xxx auto-refresh=true access-count=5000 + */ + private Map paramsKv = new HashMap<>(); + + public HintSQLInfo(String hintSQL){ + this.hintSQL = hintSQL; + } + + public String getExecSQL() { + return execSQL; + } + + public void setExecSQL(String execSQL) { + this.execSQL = execSQL; + } + + public Map getParamsKv() { + return paramsKv; + } + + public void setParamsKv(Map paramsKv) { + this.paramsKv = paramsKv; + } + + public String getFunctionName() { + return functionName; + } + + public void setFunctionName(String functionName) { + this.functionName = functionName; + } + + public String getHintSQL() { + return hintSQL; + } + + public void setHintSQL(String hintSQL) { + this.hintSQL = hintSQL; + } + + public HintHandler getHintHandler() { + return hintHandler; + } + + public void setHintHandler(HintHandler hintHandler) { + this.hintHandler = hintHandler; + } + + public boolean isCache() { + return isCache; + } + + public void setCache(boolean cache) { + isCache = cache; + } + + @Override + public String toString() { + return "HintSQLInfo{" + + "functionName='" + functionName + '\'' + + ", hintSQL='" + hintSQL + '\'' + + ", execSQL='" + execSQL + '\'' + + ", paramsKv=" + paramsKv + + '}'; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java new file mode 100644 index 0000000..3da6aa2 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java @@ -0,0 +1,79 @@ +package io.mycat.sqlcache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; + +/** + * 主要用来解析 Hint SQL + * 兼容mycat 1.6版本 新的注解方式 + * + * @author zagnix + * @create 2017-01-16 17:23 + */ + +public class HintSQLParser { + private static final Logger LOGGER = LoggerFactory.getLogger(HintSQLParser.class); + + //注解格式:/*!mycat:type=value */ sql. sql为真实的执行的sql语句 + private final static String MYCAT_HINT_START = "/*!mycat:"; + private final static String HINT_PRO_SPLIT = " "; + private final static String HINT_KV_SPLIT = "="; + + + public static HintSQLInfo parserHintSQL(String sql){ + if (sql == null) + return null; + + HintSQLInfo hintSQLInfo = new HintSQLInfo(sql); + Map hintKv = new HashMap(); + int endIndex = sql.indexOf("*/"); + + if (endIndex !=-1 ){ + String hintPart = sql.substring(0,endIndex); + hintSQLInfo.setExecSQL(sql.substring(endIndex+2)); + String kvsPart=hintPart.replace(MYCAT_HINT_START,"").replace("*/",""); + StringTokenizer token = new StringTokenizer(kvsPart,HINT_PRO_SPLIT); + + /**hint功能*/ + if (token.hasMoreElements()) { + StringTokenizer function = new StringTokenizer(token.nextToken(), HINT_KV_SPLIT); + if (function.countTokens()==2){ + String kName = function.nextToken(); + String vValue = function.nextToken(); + if (kName.equalsIgnoreCase("cachable") && vValue.equals("true")){ + hintSQLInfo.setFunctionName(kName); + hintSQLInfo.setCache(true); + } + + }else { + LOGGER.error(sql + "注解中的KV属性不对"); + return null; + } + + /**KV**/ + while (token.hasMoreElements()) { + StringTokenizer t = new StringTokenizer(token.nextToken(), HINT_KV_SPLIT); + + if (t.countTokens() == 2) { + /**TODO 限制hintSQL中kv中k具体命名 */ + hintKv.put(t.nextToken(),t.nextToken()); + }else { /**KV*/ + LOGGER.error(sql + "注解中的KV属性不对"); + return null; + } + } + hintSQLInfo.setParamsKv(hintKv); + }else { /**没有function name*/ + LOGGER.error(sql + "注解中的功能没有名字"); + return null; + } + }else { + return null; + } + return hintSQLInfo; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java new file mode 100644 index 0000000..b5a3dbb --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java @@ -0,0 +1,131 @@ +package io.mycat.sqlcache; + +import io.mycat.SQLEngineCtx; +import io.mycat.backend.MySQLBackendConnection; +import io.mycat.backend.MySQLDataSource; +import io.mycat.backend.MySQLReplicatSet; +import io.mycat.backend.callback.SQLResCacheHintHandler; +import io.mycat.beans.DNBean; +import io.mycat.engine.UserSession; +import io.mycat.front.MySQLFrontConnection; +import io.mycat.mysql.packet.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; + + +/** + * SQL 结果集缓存服务 + * + * @author zagnix + * @version 1.0 + * @create 2017-01-17 17:23 + */ + +public class SQLResultsCacheService { + private static final Logger LOGGER = LoggerFactory.getLogger(SQLResultsCacheService.class); + static class InitSQLResultsCacheService{ + public final static SQLResultsCacheService sqlResultsCacheService = new SQLResultsCacheService(); + } + + private SQLResultsCacheService(){ + } + + + + /** + * 处理Select Hint SQL 结果集函数 + * @param frontCon + * @param hintSQLInfo + * @param mySQLMessage + * @return + * @throws IOException + */ + public boolean processHintSQL(MySQLFrontConnection frontCon,HintSQLInfo hintSQLInfo, MySQLMessage mySQLMessage) throws IOException { + + + + /** + * 1. 改写sql语句，去掉前面的注释 + */ + CommandPacket command = new CommandPacket(); + command.packetId = 0; + command.command = MySQLPacket.COM_QUERY; + + /** + * 2. 获取后端DB连接池，执行rewrite sql + */ + MySQLBackendConnection existCon = null; + UserSession session=frontCon.getSession(); + ArrayList allBackCons = session.getBackendCons(); + if (!allBackCons.isEmpty()) { + existCon = allBackCons.get(0); + } + if (existCon == null || existCon.isClosed()) { + if (existCon != null) { + session.removeBackCon(existCon); + } + if (frontCon.getMycatSchema() == null){ + frontCon.writeErrMessage(1450, "No schema selected"); + return false; + } + + final DNBean dnBean = frontCon.getMycatSchema().getDefaultDN(); + final String replica = dnBean.getMysqlReplica(); + final SQLEngineCtx ctx = SQLEngineCtx.INSTANCE(); + final MySQLReplicatSet repSet = ctx.getMySQLReplicatSet(replica); + final MySQLDataSource datas = repSet.getCurWriteDH(); + /** + * 如果该sql对应后端db，没有连接池，则创建连接池部分 + */ + final MySQLBackendConnection newCon = + datas.getConnection(frontCon.getReactor(), dnBean.getDatabase(), true, null); + + /**很关键的设置前端front 与 backend session*/ + newCon.setAttachement(frontCon); + + /**设置后端连接池结果集处理handler,sqlResultCache缓存结果集类*/ + newCon.setUserCallback(new SQLResCacheHintHandler()); + + /** + * 执行sql语句 + */ + frontCon.addTodoTask(() -> { + /** + * 将数据写到后端连接池中 + */ + command.arg = hintSQLInfo.getExecSQL().getBytes(newCon.getCharset()); + newCon.getWriteDataBuffer().putBytes(command.write(newCon)); + newCon.enableWrite(false); + /** + * 新建立的连接放到连接池中 + */ + session.addBackCon(newCon); + + }); + } else { + /** + * 否则直接写到后端即可 + */ + command.arg = hintSQLInfo.getExecSQL().getBytes(existCon.getCharset()); + existCon.getWriteDataBuffer().putBytes(command.write(existCon)); + existCon.enableWrite(false); + /**设置后端连接池结果集处理handler,sqlResultCache缓存结果集类*/ + existCon.setUserCallback(new SQLResCacheHintHandler()); + } + + return true; + } + + /** + * + * @return + */ + public static SQLResultsCacheService getInstance() + { + return InitSQLResultsCacheService.sqlResultsCacheService; + } + +} From e601f1c60ba5687c50305ee5385137bc5dde85d9 Mon Sep 17 00:00:00 2001 From: "547557645@qq.com" <547557645@qq.com> Date: Sun, 22 Jan 2017 10:45:34 +0800 Subject: [PATCH 4/7] =?UTF-8?q?HintSQLParser=E4=B8=ADcachable=E6=94=B9?= =?UTF-8?q?=E6=88=90cacheable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java index 3da6aa2..4a31481 100644 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLParser.java @@ -44,7 +44,7 @@ public static HintSQLInfo parserHintSQL(String sql){ if (function.countTokens()==2){ String kName = function.nextToken(); String vValue = function.nextToken(); - if (kName.equalsIgnoreCase("cachable") && vValue.equals("true")){ + if (kName.equalsIgnoreCase("cacheable") && vValue.equals("true")){ hintSQLInfo.setFunctionName(kName); hintSQLInfo.setCache(true); } @@ -59,7 +59,7 @@ public static HintSQLInfo parserHintSQL(String sql){ StringTokenizer t = new StringTokenizer(token.nextToken(), HINT_KV_SPLIT); if (t.countTokens() == 2) { - /**TODO 限制hintSQL中kv中k具体命名 */ + /**TODO 限制hintSQL中kv中k具体命名必须在{cacheable, cache-time,auto-refresh, access-count}*/ hintKv.put(t.nextToken(),t.nextToken()); }else { /**KV*/ LOGGER.error(sql + "注解中的KV属性不对"); From 4e92a768290410e5aca1a0750a3b25581331af80 Mon Sep 17 00:00:00 2001 From: "547557645@qq.com" <547557645@qq.com> Date: Sun, 22 Jan 2017 11:23:05 +0800 Subject: [PATCH 5/7] =?UTF-8?q?SQL=20Cache=20=E7=BB=93=E6=9E=9C=E9=9B=86?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E7=BC=93=E5=AD=98=E5=88=9D?= =?UTF-8?q?=E6=AD=A5=E5=AE=9E=E7=8E=B0=201.=E6=A0=B9=E6=8D=AE=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E5=A4=84=E7=90=86=E7=9A=84sql=EF=BC=8C=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2SQLCache=E6=98=AF=E5=90=A6=E5=B7=B2=E7=BB=8F=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E4=BA=86=E3=80=82=E7=BC=93=E5=AD=98=E4=BA=86=EF=BC=8C?= =?UTF-8?q?=E5=88=99=E7=9B=B4=E6=8E=A5=E4=BB=8ECache=E4=B8=AD=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=BB=93=E6=9E=9C=E9=9B=86=E5=8F=91=E7=BB=99client?= =?UTF-8?q?=E7=AB=AF=E5=8D=B3=E5=8F=AF=202.=E6=B2=A1=E6=9C=89=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=9A=84=E6=83=85=E5=86=B5=E5=88=99=E5=B0=86sql?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=8F=91=E9=80=81=E7=9A=84=E5=90=8E=E7=AB=AF?= =?UTF-8?q?Mysql=E6=95=B0=E6=8D=AE=E5=BA=93=EF=BC=8C=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=E6=9F=A5=E8=AF=A2=EF=BC=8C=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=9A=84=E7=BB=93=E6=9E=9C=E9=9B=86=EF=BC=8C=E5=88=86=E4=BA=8C?= =?UTF-8?q?=E9=83=A8=E8=B5=B0=20=091>.=20=E7=9B=B4=E6=8E=A5=E9=80=8F?= =?UTF-8?q?=E4=BC=A0=E7=BB=99Client=20=092>.=20=E7=BC=93=E5=AD=98=E5=88=B0?= =?UTF-8?q?MyCat=E6=9C=AC=E5=9C=B0=203.=E6=A0=B9=E6=8D=AEHint=E6=B3=A8?= =?UTF-8?q?=E8=A7=A3=E7=9A=84=E5=B1=9E=E6=80=A7=EF=BC=8C=E5=86=B3=E5=AE=9A?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E6=8B=89=E5=8E=BB=E5=90=8E=E7=AB=AF=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E9=9B=86=EF=BC=8C=E6=9B=BF=E6=8D=A2=E6=8E=89?= =?UTF-8?q?=E6=97=A7=E7=9A=84=E6=95=B0=E6=8D=AE=E9=9B=86(=E5=BE=85?= =?UTF-8?q?=E5=AE=8C=E5=96=84)=204.SQL=E7=BB=93=E6=9E=9C=E9=9B=86=E6=8C=89?= =?UTF-8?q?=E7=85=A7mysql=20server=E5=8D=8F=E8=AE=AE=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E6=9C=AC=E5=9C=B0=E6=96=87=E4=BB=B6=E4=B8=AD=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=9B=9E=E6=94=BE=E3=80=82=20=09=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E6=96=B9=E5=BC=8F=E8=A7=81=EF=BC=9ASQLResCacheHintHan?= =?UTF-8?q?dler=E4=B8=ADhandleResponse=20=09=E6=9C=AC=E5=9C=B0=E5=9B=9E?= =?UTF-8?q?=E6=94=BE=E8=A7=81=EF=BC=9ASQLResultsCacheService=E4=B8=ADsqlRe?= =?UTF-8?q?sultCacheDirectClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mycat-Core/pom.xml | 8 + .../src/main/java/io/mycat/MycatCore.java | 2 +- .../MySQLBackendConnectionHandler.java | 6 +- .../callback/SQLResCacheHintHandler.java | 78 +- .../impl/AbstractSchemaSQLCommandHandler.java | 32 +- .../front/CheckUserLoginResponseCallback.java | 3 + .../front/MySQLFrontConnectionHandler.java | 4 + .../main/java/io/mycat/net2/Connection.java | 4 +- .../mycat/net2/MappedFileConDataBuffer3.java | 5 +- .../java/io/mycat/sqlcache/AddressIndex.java | 94 ++ .../java/io/mycat/sqlcache/BigSQLResult.java | 136 +++ .../main/java/io/mycat/sqlcache/CacheImp.java | 175 ++++ .../io/mycat/sqlcache/HintSQLDataLoader.java | 57 ++ .../sqlcache/HintSQLRemoveKeyListener.java | 23 + .../java/io/mycat/sqlcache/IBufferPage.java | 76 ++ .../io/mycat/sqlcache/IBufferPageFactory.java | 63 ++ .../main/java/io/mycat/sqlcache/ICache.java | 42 + .../java/io/mycat/sqlcache/IDataLoader.java | 18 + .../io/mycat/sqlcache/IRemoveKeyListener.java | 19 + .../java/io/mycat/sqlcache/ISQLResult.java | 76 ++ .../main/java/io/mycat/sqlcache/Keyer.java | 113 +++ .../java/io/mycat/sqlcache/LocatePolicy.java | 46 + .../io/mycat/sqlcache/MyCatBufferPage.java | 141 +++ .../java/io/mycat/sqlcache/PageLRUCache.java | 231 +++++ .../sqlcache/SQLResultsCacheService.java | 158 +++- .../impl/directmem/DirectMemoryFactory.java | 46 + .../impl/directmem/DirectMemorySQLResult.java | 56 ++ .../impl/mmap/MappedBufferPageFactory.java | 307 ++++++ .../impl/mmap/MappedMemFileBufferPage.java | 193 ++++ .../sqlcache/impl/mmap/MappedSQLResult.java | 890 ++++++++++++++++++ .../java/io/mycat/sqlcache/package-info.java | 7 + .../main/java/io/mycat/util/UnsafeMemory.java | 52 + .../src/main/java/io/mycat/util/Utils.java | 74 ++ .../sqlcache/MyCatBigSqlResultsCache.java | 231 +++++ .../io/mycat/sqlcache/TestHintSqlParser.java | 17 + .../io/mycat/sqlcache/testSQLResultCache.java | 128 +++ 36 files changed, 3596 insertions(+), 15 deletions(-) create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/LocatePolicy.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java create mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java create mode 100644 Mycat-Core/src/main/java/io/mycat/util/UnsafeMemory.java create mode 100644 Mycat-Core/src/main/java/io/mycat/util/Utils.java create mode 100644 Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java create mode 100644 Mycat-Core/src/test/java/io/mycat/sqlcache/TestHintSqlParser.java create mode 100644 Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java diff --git a/Mycat-Core/pom.xml b/Mycat-Core/pom.xml index e740350..b05f3d1 100644 --- a/Mycat-Core/pom.xml +++ b/Mycat-Core/pom.xml @@ -78,6 +78,14 @@ netty-buffer 4.1.7.Final + + + + com.google.guava + guava + 21.0 + + diff --git a/Mycat-Core/src/main/java/io/mycat/MycatCore.java b/Mycat-Core/src/main/java/io/mycat/MycatCore.java index c6c40cc..43a6fd4 100644 --- a/Mycat-Core/src/main/java/io/mycat/MycatCore.java +++ b/Mycat-Core/src/main/java/io/mycat/MycatCore.java @@ -79,7 +79,7 @@ public static void main(String[] args) throws IOException { connector.start(); NetSystem.getInstance().setConnector(connector); final SystemConfig sysconfig = new SystemConfig(); - sysconfig.setTraceProtocol(true); + sysconfig.setTraceProtocol(false); NetSystem.getInstance().setNetConfig(sysconfig); MySQLBackendConnectionFactory bakcMySQLFactory=new MySQLBackendConnectionFactory(); SQLEngineCtx.INSTANCE().setBackendMySQLConFactory(bakcMySQLFactory); diff --git a/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java b/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java index 001c610..e584c5f 100644 --- a/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/backend/MySQLBackendConnectionHandler.java @@ -62,6 +62,7 @@ public void handleReadEvent(MySQLBackendConnection con) throws IOException{ } length = MySQLConnection.getPacketLength(dataBuffer, offset); + if(length+offset>limit) { LOGGER.info("Not whole package :length "+length+" cur total length "+limit); @@ -72,11 +73,12 @@ public void handleReadEvent(MySQLBackendConnection con) throws IOException{ int pkgStartPos=offset; + LOGGER.info("received pkg ,length "+length+" type "+packetType+" cur total length "+limit); con.getUserCallback().handleResponse(con, dataBuffer, packetType, pkgStartPos, length); + offset += length; + dataBuffer.setReadingPos(offset); - offset += length; - dataBuffer.setReadingPos(offset); } diff --git a/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java index 763e763..8ec5985 100644 --- a/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java @@ -3,16 +3,19 @@ import io.mycat.backend.BackConnectionCallback; import io.mycat.backend.MySQLBackendConnection; import io.mycat.front.MySQLFrontConnection; +import io.mycat.mysql.packet.MySQLPacket; import io.mycat.net2.ConDataBuffer; import io.mycat.net2.Connection; import io.mycat.net2.ConnectionException; -import io.mycat.sqlcache.HintHandler; -import io.mycat.sqlcache.HintSQLInfo; +import io.mycat.sqlcache.*; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import static io.mycat.mysql.MySQLConnection.RESULT_FETCH_STATUS; +import static io.mycat.mysql.MySQLConnection.RESULT_HEADER_STATUS; + /** * SQL 结果集缓存 HintHandler * @@ -21,11 +24,18 @@ */ public class SQLResCacheHintHandler implements BackConnectionCallback,HintHandler { + private final BigSQLResult sqlResultCache; private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SQLResCacheHintHandler.class); private int filedCount = 0; + private IDataLoader loader; + private IRemoveKeyListener listener; private HintSQLInfo hintSQLInfo; - public SQLResCacheHintHandler(){ + public SQLResCacheHintHandler(HintSQLInfo hintSQLInfo, BigSQLResult sqlResultCache){ this.hintSQLInfo = hintSQLInfo; + this.sqlResultCache = sqlResultCache; + this.listener = new HintSQLRemoveKeyListener<>(); + this.loader = new HintSQLDataLoader<>(); + this.sqlResultCache.reset(); } @Override @@ -56,7 +66,67 @@ public void handleResponse(MySQLBackendConnection conn, ConDataBuffer dataBuffer ByteBuffer packetBuffer = dataBuffer.getBytes(pkgStartPos,pkgLen); frontCon.getWriteDataBuffer().putBytes(packetBuffer); frontCon.enableWrite(false); - conn.setNextStatus(packetType); + int status = conn.getState(); + switch (status) { + case Connection.STATE_IDLE: + if (packetType == MySQLPacket.COM_QUERY) { + conn.setState(Connection.STATE_IDLE); + } else if (packetType == MySQLPacket.COM_QUIT) { + conn.setState(Connection.STATE_IDLE); + }else if (sqlResultCache != null){ + /**step 1: Result Set Header Packet 列的数目*/ + LOGGER.error("step 1 =====> DB status: Result Filed Count "); + byte [] headers = new byte[packetBuffer.remaining()]; + packetBuffer.get(headers); + sqlResultCache.put(headers); + conn.setState(RESULT_HEADER_STATUS); + } + break; + case RESULT_HEADER_STATUS: + if (packetType == MySQLPacket.EOF_PACKET) { + + LOGGER.error("step 3 : EOF Packet,marker—end of field packets"); + byte [] fieldDescsEof = new byte[packetBuffer.remaining()]; + packetBuffer.get(fieldDescsEof); + sqlResultCache.put(fieldDescsEof); + + conn.setState(RESULT_FETCH_STATUS); + } else if (packetType == MySQLPacket.ERROR_PACKET) { + conn.setState(Connection.STATE_IDLE); + } else if (sqlResultCache != null) { + /**Step 2: Field Packets,列的描述信息*/ + LOGGER.error("Step 2: Field Packets"); + byte [] fieldDescs = new byte[packetBuffer.remaining()]; + packetBuffer.get(fieldDescs); + sqlResultCache.put(fieldDescs); + } + break; + case RESULT_FETCH_STATUS: + if (packetType == MySQLPacket.EOF_PACKET) { + /**Step 5:EOF Packet: marker---end of row data packets*/ + LOGGER.error("Step 5:EOF Packet: marker---end of row data packets"); + byte [] rowDataEof = new byte[packetBuffer.remaining()]; + packetBuffer.get(rowDataEof); + sqlResultCache.put(rowDataEof); + /**将sqlResultCache插入CacheService中*/ + SQLResultsCacheService.getInstance(). + cacheSQLResult(hintSQLInfo,sqlResultCache,loader,listener); + + conn.setState(Connection.STATE_IDLE); + } else if (packetType == MySQLPacket.ERROR_PACKET) { + conn.setState(Connection.STATE_IDLE); + } else if (sqlResultCache != null){ + /**Step 4: Row Data Packet{多个}: row contents 一行对应一个row packet*/ + LOGGER.error("Step 4: Row Data Packet"); + byte [] rowDatas = new byte[packetBuffer.remaining()]; + packetBuffer.get(rowDatas); + sqlResultCache.put(rowDatas); + } + break; + default: + LOGGER.warn("Error connected status.", status); + break; + } if (conn.getState()== Connection.STATE_IDLE) { diff --git a/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java b/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java index 47302f7..ddd7dd7 100644 --- a/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/engine/impl/AbstractSchemaSQLCommandHandler.java @@ -100,7 +100,7 @@ private void initDb(final MySQLFrontConnection frontCon, final ByteBuffer byteBu private void doSQLCommand(MySQLFrontConnection frontCon, ConDataBuffer dataBuffer, byte packageType, int pkgStartPos, int pkgLen) throws IOException { { - // 取得语句 + //取得语句 ByteBuffer byteBuff = dataBuffer.getBytes(pkgStartPos, pkgLen); MySQLMessage mm = new MySQLMessage(byteBuff); mm.position(5); @@ -126,6 +126,7 @@ private void doSQLCommand(MySQLFrontConnection frontCon, ConDataBuffer dataBuffe return; } + // 执行查询 SQLInfo sqlInf=new SQLInfo(); int rs = ServerParse.parse(sql,sqlInf); @@ -149,7 +150,9 @@ private void doSQLCommand(MySQLFrontConnection frontCon, ConDataBuffer dataBuffe frontCon.writeErrMessage(ErrorCode.ER_NO_DB_ERROR, "No database selected"); return; } + executeSelectSQL(frontCon, dataBuffer, packageType,pkgStartPos, pkgLen,sql); + // SelectHandler.handle(sql, con, rs >>> 8); break; case ServerParse.START: @@ -204,7 +207,7 @@ private void doSQLCommand(MySQLFrontConnection frontCon, ConDataBuffer dataBuffe } public void passThroughSQL(MySQLFrontConnection frontCon, ConDataBuffer dataBuffer, int pkgStartPos, int pkgLen) throws IOException { - // 直接透传（默认） + // 直接透传（默认）获取连接池 MySQLBackendConnection existCon = null; UserSession session=frontCon.getSession(); ArrayList allBackCons = session.getBackendCons(); @@ -220,24 +223,47 @@ public void passThroughSQL(MySQLFrontConnection frontCon, ConDataBuffer dataBuff LOGGER.error("No schema selected"); return ; } + final DNBean dnBean = frontCon.getMycatSchema().getDefaultDN(); final String replica = dnBean.getMysqlReplica(); final SQLEngineCtx ctx = SQLEngineCtx.INSTANCE(); LOGGER.debug("select a replica: {}", replica); final MySQLReplicatSet repSet = ctx.getMySQLReplicatSet(replica); final MySQLDataSource datas = repSet.getCurWriteDH(); + + /** + * 如果该sql对应后端db，没有连接池，则创建连接池部分 + */ final MySQLBackendConnection newCon = datas.getConnection(frontCon.getReactor(), dnBean.getDatabase(), true, null); + + /**很关键的设置前端front 与 backend session*/ newCon.setAttachement(frontCon); + + /**设置后端连接池结果集处理handler*/ newCon.setUserCallback(directTransCallback); + + /** + * 执行sql语句 + */ frontCon.addTodoTask(() -> { - newCon.getWriteDataBuffer().putBytes(dataBuffer.getBytes(pkgStartPos, pkgLen)); + /** + * 将数据写到后端连接池中 + */ + newCon.getWriteDataBuffer().putBytes(dataBuffer.getBytes(pkgStartPos,pkgLen)); newCon.enableWrite(false); + /** + * 新建立的连接放到连接池中 + */ session.addBackCon(newCon); }); } else { + /** + * 否则直接写到后端即可 + */ existCon.getWriteDataBuffer().putBytes(dataBuffer.getBytes(pkgStartPos, pkgLen)); existCon.enableWrite(false); + existCon.setUserCallback(directTransCallback); } } diff --git a/Mycat-Core/src/main/java/io/mycat/front/CheckUserLoginResponseCallback.java b/Mycat-Core/src/main/java/io/mycat/front/CheckUserLoginResponseCallback.java index 1ec9c37..f0a0f27 100644 --- a/Mycat-Core/src/main/java/io/mycat/front/CheckUserLoginResponseCallback.java +++ b/Mycat-Core/src/main/java/io/mycat/front/CheckUserLoginResponseCallback.java @@ -89,6 +89,9 @@ private void success(MySQLFrontConnection con, AuthPacket auth) throws IOExcepti } LOGGER.debug("charset = {}, charsetIndex = {}", charset, charsetIndex); con.setCharset(charsetIndex, charset); + + //认证成功后，修改changeCmdHandler，由CheckUserLoginResponseCallback改用 + // AbstractSchemaSQLCommandHandler处理 if (!con.setFrontSchema(auth.database)) { final String errmsg = "No Mycat Schema defined: " + auth.database; LOGGER.debug(errmsg); diff --git a/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java b/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java index badf4bd..8b4af36 100644 --- a/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/front/MySQLFrontConnectionHandler.java @@ -89,6 +89,8 @@ public void handleReadEvent(final MySQLFrontConnection cnxn) throws IOException{ final byte packetType = buffer.getByte(offset + MySQLConnection.msyql_packetHeaderSize); final int pkgStartPos = offset; + + // trace-protocol-packet // @author little-pan // @since 2016-09-29 @@ -102,6 +104,8 @@ public void handleReadEvent(final MySQLFrontConnection cnxn) throws IOException{ offset += length; buffer.setReadingPos(offset); + + } } diff --git a/Mycat-Core/src/main/java/io/mycat/net2/Connection.java b/Mycat-Core/src/main/java/io/mycat/net2/Connection.java index a8e36dc..2a2f7d0 100644 --- a/Mycat-Core/src/main/java/io/mycat/net2/Connection.java +++ b/Mycat-Core/src/main/java/io/mycat/net2/Connection.java @@ -286,11 +286,11 @@ public void register(Selector selector, ReactorBufferPool myBufferPool) throws I //TODO /**使用MyCatMemoryAllocator分配Direct Buffer,再进行SocketChannel通信时候， - * NIO读写都会减少一次数据的拷贝,而使用FileChanel与SocketChannel数据交换时 + * 网络读写都会减少一次数据的拷贝,而使用FileChanel与SocketChannel数据交换时 * 底层最终还是生成一个临时的Direct Buffer，用临时Direct Buffer写入或者读SocketChannel中 * 后面考虑会使用netty中ByteBuf中的DirectBuffer进行网络IO通信。效率更高 * */ - this.readDataBuffer =new MappedFileConDataBuffer3(maprFileName); // 2 ,3 + this.readDataBuffer =new MappedFileConDataBuffer(maprFileName); // 2 ,3 this.writeDataBuffer=new MappedFileConDataBuffer3(mapwFileName); //存在bug暂不启用，以后统一是ByteBuf作为buffer进行NIO网络通信。 // this.readDataBuffer = new ByteBufConDataBuffer(4096,16*1024*1024); diff --git a/Mycat-Core/src/main/java/io/mycat/net2/MappedFileConDataBuffer3.java b/Mycat-Core/src/main/java/io/mycat/net2/MappedFileConDataBuffer3.java index 50e29b8..aa9410b 100644 --- a/Mycat-Core/src/main/java/io/mycat/net2/MappedFileConDataBuffer3.java +++ b/Mycat-Core/src/main/java/io/mycat/net2/MappedFileConDataBuffer3.java @@ -31,6 +31,7 @@ import java.nio.channels.FileChannel; import java.nio.channels.SocketChannel; +import io.mycat.memalloc.MyCatMemoryAllocator; import sun.misc.Unsafe; import sun.nio.ch.FileChannelImpl; /** @@ -168,7 +169,9 @@ public ByteBuffer getBytes(int index,int length) throws IOException { } @Override public ByteBuffer beginWrite(int length) { - ByteBuffer copyBuf=ByteBuffer.allocate(length); + //TODO + //ByteBuffer copyBuf=ByteBuffer.allocate(length); + ByteBuffer copyBuf = MyCatMemoryAllocator.getINSTANCE().directBuffer(length).nioBuffer(0,length); return copyBuf; } diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java new file mode 100644 index 0000000..c3d418e --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java @@ -0,0 +1,94 @@ +package io.mycat.sqlcache; + +import io.mycat.util.UnsafeMemory; +import sun.misc.Unsafe; + +/** + * cache padding from disruptor + * + * @author zagnix + * @create 2016-11-18 10:17 + */ + +class LhsPadding +{ + protected long p1, p2, p3, p4, p5, p6, p7; +} + +class Value extends LhsPadding +{ + protected volatile long value; +} + +class RhsPadding extends Value +{ + protected long p9, p10, p11, p12, p13, p14, p15; +} + +public class AddressIndex extends RhsPadding { + static final long INITIAL_VALUE = -1L; + private static final Unsafe UNSAFE = UnsafeMemory.getUnsafe(); + private static final long VALUE_OFFSET; + + public AddressIndex() { + this(INITIAL_VALUE); + } + + public AddressIndex(long initialValue) { + UNSAFE.putOrderedLong(this, VALUE_OFFSET, initialValue); + } + + public long get() { + return this.value; + } + + public void set(long value) { + UNSAFE.putOrderedLong(this, VALUE_OFFSET, value); + } + + public void setVolatile(long value) { + UNSAFE.putLongVolatile(this, VALUE_OFFSET, value); + } + + /** + * 内存地址值(通旧值this所在对象加上字节偏量VALUE_OFFSET找到所在的内存段) + * 期望值: expectedValue + * 新值: newValue + * 如果 内存地址值 == 期望值 + * 则将新的值写入到内存地址中。 + * return true + * else + * return false + * @param expectedValue + * @param newValue + * @return + */ + public boolean compareAndSet(long expectedValue, long newValue) { + return UNSAFE.compareAndSwapLong(this, VALUE_OFFSET, expectedValue, newValue); + } + + + public long incrementAndGet() { + return this.addAndGet(1L); + } + + public long addAndGet(long increment) { + long currentValue; + long newValue; + do { + currentValue = this.get(); + newValue = currentValue + increment; + } while(!this.compareAndSet(currentValue,newValue)); + return newValue; + } + public String toString() { + return Long.toString(this.get()); + } + static { + try { + VALUE_OFFSET = UNSAFE.objectFieldOffset(Value.class.getDeclaredField("value")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java new file mode 100644 index 0000000..d8f0cba --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java @@ -0,0 +1,136 @@ +package io.mycat.sqlcache; + + +import io.mycat.sqlcache.impl.mmap.MappedSQLResult; + +import java.io.IOException; +import java.util.Iterator; + +import static com.google.common.hash.Hashing.murmur3_32; + +/** + * SQL大结果集缓存 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-27 18:48 + */ + +public class BigSQLResult implements Iterator { + + private ISQLResult sqlResult; + private String cacheDir; + + public BigSQLResult(LocatePolicy locatePolicy, String sql, int pageSize){ + /** + * Core - DirectMemory + * Normal - mmap + */ + this.cacheDir = "sqlcache/"; + String sqlkey = ""+murmur3_32().hashUnencodedChars(sql); + + if (locatePolicy.equals(LocatePolicy.Normal)){ + try { + sqlResult = new MappedSQLResult(cacheDir,sqlkey,pageSize); + } catch (IOException e) { + e.printStackTrace(); + } + }else if(locatePolicy.equals(LocatePolicy.Core)){ + + } + } + + /** + * 添加一条sql二进制数据到Cache存储中 + * @param data + */ + public void put(byte [] data){ + try { + sqlResult.put(data); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 数据是否为空 + * @return + */ + public boolean isEmpty(){ + return sqlResult.isEmpty(); + } + + /** + * 还有下一条sql结果集？ + * @return + */ + public boolean hasNext() { + return !sqlResult.isEmpty(); + } + + /** + * 取下一条sql结果集 + * + * @return + */ + public byte[] next() { + try { + return sqlResult.next(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + /** + * Iterator interface + */ + public void remove() { + //TODO + } + + /** + * 主动将内存映射文件的数据刷到磁盘中 + */ + public void flush(){ + sqlResult.flush(); + } + + /** + * sql 结果集大小 + * @return + */ + public long size(){ + return sqlResult.size(); + } + + /** + * 从 PageLRUCache中移除Page，并执行unmap操作，并删除对于文件 + */ + public void removeAll(){ + try { + sqlResult.removeAll(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 从 PageLRUCache中移除Page，并执行unmap操作，但不删除文件 + * 下次运行时候可以读取文件内容 + */ + public void recycle(){ + try { + sqlResult.recycle(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 复位读位置为0，从头开始读取sql结果集 + */ + public void reset(){ + sqlResult.reset(); + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java new file mode 100644 index 0000000..2d8f781 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java @@ -0,0 +1,175 @@ +package io.mycat.sqlcache; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + + +/** + * ICache实现类 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-30 16:53 + */ + +public class CacheImp implements ICache { + + private final static Logger logger + = LoggerFactory.getLogger(CacheImp.class); + + public static final long DEFAULT_TTL = 5 * 1000; + public static final int ACCESS_COUNT = 500; + private static final int REMOVE_ACTION = 1; + private static final int RELOAD_ACTION = 2; + + private final Map cacheMap; + private final Map> cacheKeyers; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Lock readLock = lock.readLock(); + private final Lock writeLock = lock.writeLock(); + + + /** + * 异步线程，按照cache规则生效 + * 1.缓存时间到期，进行del + * 2.如果在缓存时间内，访问次数超过N次，重新从DB后台load数据，更新Value + */ + private static final ThreadFactory threadFactory = + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("async-op-cache ").build(); + private static final ScheduledExecutorService scheduleAtFixedRate = + new ScheduledThreadPoolExecutor(1,threadFactory); + private final Set> keysToDel = new HashSet>(); + private final Set> keysToReload= new HashSet>(); + + + public CacheImp(){ + cacheMap = new ConcurrentHashMap(); + cacheKeyers = new ConcurrentHashMap>(); + + scheduleAtFixedRate.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + keysToDel.clear(); + Set keySets = cacheKeyers.keySet(); + + long currentTime = System.currentTimeMillis(); + for (K key:keySets) { + Keyer k = cacheKeyers.get(key); + long diff = currentTime-k.getLastAccessTime(); + if (diff>k.getCacheTTL()){ + keysToDel.add(k); + }else if (diff<=k.getCacheTTL() && k.getRefCount().get()>=k.getAccessCount()){ + keysToReload.add(k); + } + } + + /** + * key 失效,回调remove接口 + */ + for (Keyer key:keysToDel) { + cacheMap.remove(key.getKey()); + if(key.getRemoveKeyListener() !=null){ + key.getRemoveKeyListener().removeNotify(key.getKey(),key.getValue()); + } + cacheKeyers.remove(key.getKey()); + } + + + /** + * 回调用户reload接口 + */ + for (Keyer key:keysToReload) { + /** + * reload 设计为异步加载 + */ + if(key.getiDataLoader() !=null){ + key.getiDataLoader().reload(key); + } + } + } + },0,DEFAULT_TTL, TimeUnit.SECONDS); + } + + /** + * put (k，v) to map + * + * @param key + * @param value + * @param keyer + */ + @Override + public void put(K key, V value,Keyer keyer) { + cacheMap.put(key,value); + cacheKeyers.put(key,keyer); + } + + /** + * get value + * + * @param key + * @return + */ + @Override + public V get(K key) { + + try { + readLock.lock(); + Keyer keyer = cacheKeyers.get(key); + + if (keyer != null){ + keyer.setLastAccessTime(System.currentTimeMillis()); + keyer.getRefCount().decrementAndGet(); + } + + return cacheMap.get(key); + + }finally { + readLock.unlock(); + } + + + } + + /** + * remove key + * + * @param key + */ + @Override + public void remove(K key) { + Keyer keyer = cacheKeyers.get(key); + + if (keyer.getRemoveKeyListener()!=null){ + keyer.getRemoveKeyListener(). + removeNotify(keyer.getKey(),keyer.getValue()); + } + cacheKeyers.remove(key); + cacheMap.remove(key); + } + + + /** + * remove all map's element + */ + @Override + public void removeALL() { + for (K key:cacheKeyers.keySet()) { + Keyer keyer = cacheKeyers.get(key); + if (keyer.getRemoveKeyListener()!=null){ + keyer.getRemoveKeyListener().removeNotify(keyer.getKey(),keyer.getValue()); + } + } + cacheKeyers.clear(); + cacheMap.clear(); + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java new file mode 100644 index 0000000..d44e163 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java @@ -0,0 +1,57 @@ +package io.mycat.sqlcache; + +import com.google.common.util.concurrent.*; + +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; + +import static com.google.common.hash.Hashing.murmur3_32; + +/** + * SQL 异步加载结果集类 + * + * @author zagnix + * @create 2017-01-20 15:13 + */ + +public class HintSQLDataLoader implements IDataLoader{ + /** + * 根据sql,异步从后台DB reload数据，替换旧值 + * @param keyer + * @return + */ + @Override + public V reload(Keyer keyer) { + ListeningExecutorService executor = + MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()); + + final ListenableFuture listenableFuture = executor + .submit(new Callable() { + @Override + public BigSQLResult call() throws Exception { + //TODO + String sql = keyer.getSql(); + String sqlkey = keyer.getLastAccessTime() +"_"+murmur3_32().hashUnencodedChars(sql); + BigSQLResult sqlResultCache + = new BigSQLResult(LocatePolicy.Normal,sqlkey,32*1024*1024); + + + return sqlResultCache; + } + }); + + Futures.addCallback(listenableFuture, new FutureCallback() { + @Override + public void onSuccess(BigSQLResult bigSQLResult) { + //TODO + } + + @Override + public void onFailure(Throwable t) { + //TODO + } + } + ); + return null; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java new file mode 100644 index 0000000..5fa43e6 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java @@ -0,0 +1,23 @@ +package io.mycat.sqlcache; + +/** + * HintSQL 结果集被移除时回调类 + * + * @author zagnix + * @create 2017-01-20 15:14 + */ + +public class HintSQLRemoveKeyListener implements IRemoveKeyListener{ + /** + * key 失效，做清理工作 + * + * @param key + * @param value + */ + @Override + public void removeNotify(K key, V value) { + if (value !=null){ + value.removeAll(); + } + } +} \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java new file mode 100644 index 0000000..704b01b --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java @@ -0,0 +1,76 @@ +package io.mycat.sqlcache; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Cache Page接口 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-27 18:49 + */ + +public interface IBufferPage { + + /** + * get bytes from Thread Local + * + * @param position + * @param length + * @return + */ + public byte[] getBytes(int position, int length); + + /** + * ByteBuffer slice from Thread Local + * + * @param position + * @param limit + * @return + */ + public ByteBuffer slice(int position, int limit); + + /** + * get ByteBuffer from Thread Local + * + * @param position + * @return + */ + public ByteBuffer getLocalByteBuffer(int position); + + + /** + * 将数据刷到磁盘 + */ + public void flush(); + + + /** + * 页号 + * + * @return + */ + long getPageIndex(); + + + /** + * 设置页有数据被写入了，属于’脏页‘ + * + * @param dirty + */ + void setDirty(boolean dirty); + + + /** + * 页回收 + */ + public void recycle() throws IOException; + + /** + * 页是否被回收了 + * + * @return + */ + boolean isRecycled(); +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java new file mode 100644 index 0000000..de077e7 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java @@ -0,0 +1,63 @@ +package io.mycat.sqlcache; + +import java.io.IOException; +import java.util.Set; + +/** + * Buffer Page Factory 接口 + * + * @author zagnix + * @create 2016-12-27 22:59 + */ +public interface IBufferPageFactory { + + /** + * 根据index得到一页 + * + * @param index + * @return + * @throws IOException + */ + public MyCatBufferPage acquirePage(long index) throws IOException; + + /** + * 回收Page + * + * @param index + */ + public void releasePage(long index); + + /** + * 删除一页 + * @param index + * @throws IOException + */ + public void deletePage(long index) throws IOException; + + + /** + * 刷磁盘 + */ + public void flush(); + + /** + * 删除所有页 + * @throws IOException + */ + public void deleteAllPages() throws IOException; + + + /** + * 得到所有的页 + * @return + */ + public Set getExistingPageIndexSet(); + + + /** + * 从cache中移除所有的页 + * @throws IOException + */ + public void releaseCachedPages() throws IOException; + +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java new file mode 100644 index 0000000..87c1e7e --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java @@ -0,0 +1,42 @@ +package io.mycat.sqlcache; + +/** + * 定制化 Cache接口 + * 主要实现SQL结果集缓存规则 + * 1.缓存时间 + * 2.在缓存时间内，访问次数超过N次，重新从DB后台load数据，更新Value + * @author zagnix + * @version 1.0 + * @create 2016-12-30 11:26 + */ +public interface ICache { + + /** + * + * @param key + * @param value + * @param keyer + */ + public void put(final K key, final V value, final Keyer keyer); + + /** + * + * @param key + * @return + */ + public V get(final K key); + + + /** + * + * @param key + */ + public void remove(final K key); + + /** + * + */ + public void removeALL(); + + +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java new file mode 100644 index 0000000..9599748 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java @@ -0,0 +1,18 @@ +package io.mycat.sqlcache; + +/** + * Value 数据加载接口 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-30 16:44 + */ +public interface IDataLoader{ + + /** + * Key失效，时候重新异步reload数据 + * @param keyer + * @return + */ + public V reload(Keyer keyer); +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java new file mode 100644 index 0000000..f50b7a7 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java @@ -0,0 +1,19 @@ +package io.mycat.sqlcache; + +/** + * key 被移除时，回调该接口 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-30 16:47 + */ +public interface IRemoveKeyListener { + + /** + * 当Key被移除时，回调该接口 + * + * @param key + * @param value + */ + public void removeNotify(K key, V value); +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java new file mode 100644 index 0000000..e78c886 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java @@ -0,0 +1,76 @@ +package io.mycat.sqlcache; + +import java.io.IOException; + +/** + * SQLResult 存放 实现接口 + * @author zagnix + * @create 2016-11-18 16:46 + */ +public interface ISQLResult { + + /** + * 该将sql row的二进制数据，存放到Cache中 + * @param data + * @return + * @throws IOException + */ + public long put(byte[] data) throws IOException; + + /** + * 获取一条sql row的二进制数据 + * @return + * @throws IOException + */ + public byte[] next() throws IOException; + + /** + * 根据index得到一条SQL row的二进制数据 + * @param index + * @return + * @throws IOException + */ + public byte[] get(long index) throws IOException; + + + /** + * SQL结果集条数 + * @return + */ + public long size(); + + /** + * 是否没有缓存SQL结果集 + * @return + */ + public boolean isEmpty() ; + + /** + * Cache是否已经满了 + * @return + */ + public boolean isFull(); + + /** + * 将SQL结果集刷入磁盘 + */ + public void flush(); + + /** + * 删除数据页和索引页文件 + * @throws IOException + */ + public void removeAll() throws IOException; + + /** + * 将所有内存映射文件unmap + * @throws IOException + */ + public void recycle() throws IOException; + + + /** + * 复位读的位置0，即从头开始读取SQL结果集 + */ + public void reset(); +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java new file mode 100644 index 0000000..7d96c46 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java @@ -0,0 +1,113 @@ +package io.mycat.sqlcache; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * key cache生效规则 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-30 16:48 + */ + +public class Keyer { + private long cacheTTL=0; + private long lastAccessTime=0; + + /** + * TTL 时间内访问次数 + */ + private long accessCount=0; + private AtomicLong refCount = new AtomicLong(0); + + + private boolean autoRefresh = false; + + private String sql; + private K key; + private V value; + + private IDataLoader iDataLoader = null; + private IRemoveKeyListener removeKeyListener = null; + + public long getCacheTTL() { + return cacheTTL; + } + + public void setCacheTTL(long cacheTTL) { + this.cacheTTL = cacheTTL; + } + + public long getLastAccessTime() { + return lastAccessTime; + } + + public void setLastAccessTime(long lastAccessTime) { + this.lastAccessTime = lastAccessTime; + } + + public AtomicLong getRefCount() { + return refCount; + } + + public void setRefCount(AtomicLong refCount) { + this.refCount = refCount; + } + + public K getKey() { + return key; + } + + public void setKey(K key) { + this.key = key; + } + + public V getValue() { + return value; + } + + public void setValue(V value) { + this.value = value; + } + + public IDataLoader getiDataLoader() { + return iDataLoader; + } + + public void setiDataLoader(IDataLoader iDataLoader) { + this.iDataLoader = iDataLoader; + } + + public IRemoveKeyListener getRemoveKeyListener() { + return removeKeyListener; + } + + public void setRemoveKeyListener(IRemoveKeyListener removeKeyListener) { + this.removeKeyListener = removeKeyListener; + } + + public long getAccessCount() { + return accessCount; + } + + public void setAccessCount(long accessCount) { + this.accessCount = accessCount; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + + public boolean isAutoRefresh() { + return autoRefresh; + } + + public void setAutoRefresh(boolean autoRefresh) { + this.autoRefresh = autoRefresh; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/LocatePolicy.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/LocatePolicy.java new file mode 100644 index 0000000..d22584b --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/LocatePolicy.java @@ -0,0 +1,46 @@ +package io.mycat.sqlcache; + +/** + * 策略信息 +* 源文件名：LocatePolicy.java +* 文件版本：1.0.0 +* 创建作者：Think +* 创建日期：2016年12月27日 +* 修改作者：Think +* 修改日期：2016年12月27日 +* 文件描述：TODO +* 版权所有：Copyright 2016 zjhz, Inc. All Rights Reserved. +*/ +public enum LocatePolicy { + + /** + * 使用本地的内存进行缓存的策略 + * @字段说明 Core + */ + Core(1), + + /** + * 使用文件进行映射的缓存的策略信息 + * @字段说明 Normal + */ + Normal(2); + + /** + * 策略信息 + * @字段说明 policy + */ + private int policy; + + public int getPolicy() { + return policy; + } + + public void setPolicy(int policy) { + this.policy = policy; + } + + private LocatePolicy(int policy) { + this.policy = policy; + } + +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java new file mode 100644 index 0000000..1dc0dda --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java @@ -0,0 +1,141 @@ +package io.mycat.sqlcache; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Mycat Buffer Page 抽象类 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-27 18:49 + */ + +public abstract class MyCatBufferPage implements IBufferPage { + private final static Logger logger = + LoggerFactory.getLogger(MyCatBufferPage.class); + + protected volatile boolean dirty = false; + protected volatile boolean recycled = false; + + /** + * Thread Local Buffer Page + */ + protected ThreadLocalByteBuffer threadLocalByteBuffer; + + /** + * Page 上次访问时间 + */ + protected AtomicLong lastAccessedTimestamp; + + /** + * Page 访问引用计数 + */ + protected AtomicLong refCount = new AtomicLong(0); + + + /** + * 缓存时间 + */ + protected long cacheTTL = 0L; + + public MyCatBufferPage(ByteBuffer byteBuffer,long cacheTTL){ + this.lastAccessedTimestamp = new AtomicLong(System.currentTimeMillis()); + this.threadLocalByteBuffer = new ThreadLocalByteBuffer(byteBuffer); + this.cacheTTL = cacheTTL; + } + + + /** + * Page 是 '脏页' + * @return + */ + + public boolean isDirty() { + return dirty; + } + + + /** + * 设置 Page 是否 是 ‘脏页’ + * @param dirty + */ + public void setDirty(boolean dirty) { + this.dirty = dirty; + } + + /** + * Page 是否被 unmap了 + * @return + */ + public boolean isRecycled() { + return recycled; + } + + public void setRecycled(boolean recycled) { + this.recycled = recycled; + } + + public ThreadLocalByteBuffer getThreadLocalByteBuffer() { + return threadLocalByteBuffer; + } + + public void setThreadLocalByteBuffer(ThreadLocalByteBuffer threadLocalByteBuffer) { + this.threadLocalByteBuffer = threadLocalByteBuffer; + } + + public AtomicLong getLastAccessedTimestamp() { + return lastAccessedTimestamp; + } + + public void setLastAccessedTimestamp(AtomicLong lastAccessedTimestamp) { + this.lastAccessedTimestamp = lastAccessedTimestamp; + } + + public AtomicLong getRefCount() { + return refCount; + } + + public void setRefCount(AtomicLong refCount) { + this.refCount = refCount; + } + + public long getCacheTTL() { + return cacheTTL; + } + + public void setCacheTTL(long cacheTTL) { + this.cacheTTL = cacheTTL; + } + + + /** + * Thread Local ByteBuffer + */ + protected static class ThreadLocalByteBuffer extends ThreadLocal { + private ByteBuffer byteBuffer; + + public ThreadLocalByteBuffer(ByteBuffer src) { + byteBuffer = src; + } + + public ByteBuffer getByteBuffer() { + return byteBuffer; + } + + public void setByteBuffer(ByteBuffer byteBuffer) { + this.byteBuffer = byteBuffer; + } + + + @Override + protected synchronized ByteBuffer initialValue() { + ByteBuffer bbuffer = byteBuffer.duplicate(); + return bbuffer; + } + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java new file mode 100644 index 0000000..78a951f --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java @@ -0,0 +1,231 @@ +package io.mycat.sqlcache; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * 通过LRU机制 Cache Buffer Page，提供高访问速度 + * @author zagnix + * @create 2016-12-02 14:37 + */ + +public class PageLRUCache { + private final static Logger logger = LoggerFactory.getLogger(PageLRUCache.class); + + public static final long DEFAULT_TTL = 5 * 1000; + + private final Map map; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private final Lock readLock = lock.readLock(); + private final Lock writeLock = lock.writeLock(); + + /** + * 异步线程 + */ + private static final ExecutorService executorService + = Executors.newCachedThreadPool(); + + private final Set toDisk = new HashSet(); + + public PageLRUCache() { + map = new HashMap(); + } + + /** + * 向Cache添加一个page，会触发Page Sweep操作 + * @param key + * @param value + * @param ttlInMilliSeconds + */ + public void put(Long key, MyCatBufferPage value, long ttlInMilliSeconds) { + Collection values = null; + try { + writeLock.lock(); + values = pageSweep(); + if (values != null && values.contains(value)) { + values.remove(value); + } + value.getLastAccessedTimestamp().set(System.currentTimeMillis()); + map.put(key, value); + } finally { + writeLock.unlock(); + } + + + if (values != null && values.size() > 0) { + executorService.execute(new Task(values)); + } + } + + public void put(Long key,MyCatBufferPage value) { + this.put(key, value, DEFAULT_TTL); + } + + + /** + * 根据Page Index 得到对应的 Page + * 同时会设置该Page的访问时间，和引用计数+1 + * @param key + * @return + */ + public MyCatBufferPage get(final Long key) { + try { + readLock.lock(); + MyCatBufferPage value = map.get(key); + if (value != null) { + value.getLastAccessedTimestamp().set(System.currentTimeMillis()); + value.getRefCount().incrementAndGet(); + } + return map.get(key); + } finally { + readLock.unlock(); + } + } + + /** + * 将上次访问时间超过Cache TTL 和 引用数=0的Page + * 存放到collection中，供put调用时候，异步从内存 + * 中刷到磁盘上 + * @return + */ + + private Collection pageSweep() { + Collection values = null; + toDisk.clear(); + Set keys = map.keySet(); + long currentTimeMillis = System.currentTimeMillis(); + + for(Long key: keys) { + MyCatBufferPage v = map.get(key); + if (v.getRefCount().get() <= 0 + && (currentTimeMillis - v.getLastAccessedTimestamp().get()) > v.getCacheTTL()) { + toDisk.add(key); + } + } + + if (toDisk.size() > 0) { + values = new HashSet(); + for(Long key : toDisk) { + MyCatBufferPage v = map.remove(key); + values.add(v); + } + } + + return values; + } + + /** + * 引用计数减1 + * @param key + */ + + public void release(final Long key) { + try { + readLock.lock(); + MyCatBufferPage value = map.get(key); + if (value != null) { + value.getRefCount().decrementAndGet(); + } + } finally { + readLock.unlock(); + } + } + + /** + * 缓存page的大小 + * @return + */ + public int size() { + try { + readLock.lock(); + return map.size(); + } finally { + readLock.unlock(); + } + } + + + /** + * 将所有缓存在内存中Page 数据，刷到磁盘中去 + * @throws IOException + */ + public void removeAll() throws IOException { + try { + writeLock.lock(); + Collection values = map.values(); + if (values != null && values.size() > 0) { + for(MyCatBufferPage v : values) { + v.recycle(); + } + } + map.clear(); + } finally { + writeLock.unlock(); + } + + } + + + /** + * 将index Page 数据，刷到磁盘中去 + * @param key + * @return + * @throws IOException + */ + + public MyCatBufferPage remove(final Long key) throws IOException { + try { + writeLock.lock(); + MyCatBufferPage value = map.remove(key); + if (value != null) { + value.recycle(); + } + return value; + } finally { + writeLock.unlock(); + } + } + + + + + public Collection getValues() { + try { + readLock.lock(); + return map.values(); + } finally { + readLock.unlock(); + } + } + + + /** + * 异步操作task，负责将内存中的页数据，刷到磁盘中 + */ + private static class Task implements Runnable { + Collection values; + public Task(Collection values) { + this.values = values; + } + public void run() { + for(MyCatBufferPage v : values) { + try { + if (v != null) { + v.recycle(); + } + } catch (IOException e) { + } + } + } + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java index b5a3dbb..fb03bcd 100644 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java @@ -8,13 +8,19 @@ import io.mycat.beans.DNBean; import io.mycat.engine.UserSession; import io.mycat.front.MySQLFrontConnection; +import io.mycat.mysql.MySQLConnection; import io.mycat.mysql.packet.*; +import io.mycat.net2.Connection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; +import static com.google.common.hash.Hashing.murmur3_32; +import static io.mycat.mysql.MySQLConnection.RESULT_FETCH_STATUS; +import static io.mycat.mysql.MySQLConnection.RESULT_HEADER_STATUS; /** * SQL 结果集缓存服务 @@ -26,13 +32,64 @@ public class SQLResultsCacheService { private static final Logger LOGGER = LoggerFactory.getLogger(SQLResultsCacheService.class); + private final CacheImp sqlResultCacheImp; static class InitSQLResultsCacheService{ public final static SQLResultsCacheService sqlResultsCacheService = new SQLResultsCacheService(); } private SQLResultsCacheService(){ + sqlResultCacheImp = new CacheImp(); } + /** + * 将sql结果集缓存起来 + * + * @param hintSQLInfo Hint SQL相关信息 + * @param bigSQLResult sql结果集缓存 + * @param loader 结果集load or reload 接口 + * @param listener key被移除时，调用的接口 + */ + public void cacheSQLResult(HintSQLInfo hintSQLInfo,BigSQLResult bigSQLResult,IDataLoader loader, + IRemoveKeyListener listener){ + /** + * cache-time=xxx auto-refresh=true access-count=5000 + */ + String key = "" + murmur3_32().hashUnencodedChars(hintSQLInfo.getExecSQL()); + + Keyer keyer = new Keyer(); + keyer.setSql(hintSQLInfo.getExecSQL()); + keyer.setKey(key); + keyer.setValue(bigSQLResult); + keyer.setCacheTTL(Integer.valueOf(hintSQLInfo.getParamsKv().get("cache-time"))); + keyer.setAccessCount(Integer.valueOf(hintSQLInfo.getParamsKv().get("access-count"))); + keyer.setAutoRefresh(Boolean.valueOf(hintSQLInfo.getParamsKv().get("auto-refresh"))); + keyer.setRemoveKeyListener(listener); + keyer.setiDataLoader(loader); + sqlResultCacheImp.put(key,bigSQLResult,keyer); + } + /** + * 获取sql语句，已经缓存的结果集 + * @param sql sql 语句 + * @return + */ + public BigSQLResult getSQLResult(String sql){ + String key = "" + murmur3_32().hashUnencodedChars(sql); + return sqlResultCacheImp.get(key); + } + + + /** + * 获取sql语句，已经缓存的结果集 + * + * @param sql sql 语句 + */ + public void remove(String sql){ + + String key = "" + murmur3_32().hashUnencodedChars(sql); + sqlResultCacheImp.remove(key); + } + + /** @@ -45,7 +102,25 @@ private SQLResultsCacheService(){ */ public boolean processHintSQL(MySQLFrontConnection frontCon,HintSQLInfo hintSQLInfo, MySQLMessage mySQLMessage) throws IOException { + /** + * SQL Cache 结果集缓存框架实现： + * 1.根据解析处理的sql，查询SQLCache是否已经缓存了。缓存了，则直接从Cache中获取结果集发给client端即可 + * 2.没有缓存的情况则将sql执行发送的后端Mysql数据库，执行并发查询，返回的结果集，分二部走 + * 1>. 直接透传给Client + * 2>. 缓存到MyCat本地 + * 3.根据Hint注解的属性，决定异步拉去后端的结果集，替换掉旧的数据集 + */ + BigSQLResult sqlResultCache = getSQLResult(hintSQLInfo.getExecSQL()); + if (sqlResultCache != null){ + LOGGER.error(hintSQLInfo.getExecSQL() + ":====>>>> Use Local Cache SQL Resuls"); + sqlResultCacheDirectClient(frontCon,sqlResultCache); + return true; + }else { + /**从后端拉取数据进行缓存*/ + sqlResultCache = + new BigSQLResult(LocatePolicy.Normal,hintSQLInfo.getExecSQL(),32*1024*1024/**TODO*/); + } /** * 1. 改写sql语句，去掉前面的注释 @@ -87,7 +162,7 @@ public boolean processHintSQL(MySQLFrontConnection frontCon,HintSQLInfo hintSQLI newCon.setAttachement(frontCon); /**设置后端连接池结果集处理handler,sqlResultCache缓存结果集类*/ - newCon.setUserCallback(new SQLResCacheHintHandler()); + newCon.setUserCallback(new SQLResCacheHintHandler(hintSQLInfo,sqlResultCache)); /** * 执行sql语句 @@ -113,12 +188,91 @@ public boolean processHintSQL(MySQLFrontConnection frontCon,HintSQLInfo hintSQLI existCon.getWriteDataBuffer().putBytes(command.write(existCon)); existCon.enableWrite(false); /**设置后端连接池结果集处理handler,sqlResultCache缓存结果集类*/ - existCon.setUserCallback(new SQLResCacheHintHandler()); + existCon.setUserCallback(new SQLResCacheHintHandler(hintSQLInfo,sqlResultCache)); } return true; } + /** + * 处理Local Cache结果集,直接返回给Front Connection端 + * @param frontConn + * @param sqlResultCache + */ + private void sqlResultCacheDirectClient(MySQLFrontConnection frontConn,BigSQLResult sqlResultCache) throws IOException { + sqlResultCache.reset(); + int filedCount = 0; + int status = Connection.STATE_IDLE; + while (sqlResultCache.hasNext()){ + byte [] datas = sqlResultCache.next(); + byte packetType = datas[MySQLConnection.msyql_packetHeaderSize]; + ByteBuffer packetBuffer = ByteBuffer.wrap(datas); + frontConn.getWriteDataBuffer().putBytes(packetBuffer); + frontConn.enableWrite(false); + + if (false) { + switch (status) { + case Connection.STATE_IDLE: + if (packetType == MySQLPacket.COM_QUERY) { + status = Connection.STATE_IDLE; + } else if (packetType == MySQLPacket.COM_QUIT) { + status = (Connection.STATE_IDLE); + } else if (sqlResultCache != null) { + /**step 1: Result Set Header Packet 列的数目*/ + status = RESULT_HEADER_STATUS; + ResultSetHeaderPacket resultSetHeaderPacket = new ResultSetHeaderPacket(); + resultSetHeaderPacket.read(packetBuffer); + filedCount = resultSetHeaderPacket.fieldCount; + LOGGER.error("step 1 =====> DB status: Result Filed Count " + filedCount); + + } + break; + case RESULT_HEADER_STATUS: + if (packetType == MySQLPacket.EOF_PACKET) { + LOGGER.error("step 3 : EOF Packet,marker—end of field packets"); + status = RESULT_FETCH_STATUS; + } else if (packetType == MySQLPacket.ERROR_PACKET) { + status = (Connection.STATE_IDLE); + } else if (sqlResultCache != null) { + /**Step 2: Field Packets,列的描述信息*/ + + FieldPacket fieldPacket = new FieldPacket(); + fieldPacket.read(packetBuffer); + //frontConn.getChannel().write(packetBuffer); + + LOGGER.error("Step 2: Field Packets"); + + } + break; + case RESULT_FETCH_STATUS: + if (packetType == MySQLPacket.EOF_PACKET) { + /**Step 5:EOF Packet: marker---end of row data packets*/ + LOGGER.error("Step 5:EOF Packet: marker---end of row data packets"); + status = Connection.STATE_IDLE; + break; + } else if (packetType == MySQLPacket.ERROR_PACKET) { + status = (Connection.STATE_IDLE); + } else if (sqlResultCache != null) { + /**Step 4: Row Data Packet{多个}: row contents 一行对应一个row packet*/ + + LOGGER.error("Step 4: Row Data Packet"); + RowDataPacket rowDataPacket = new RowDataPacket(filedCount); + rowDataPacket.read(packetBuffer); + for (int i = 0; i < filedCount; i++) { + byte[] src = rowDataPacket.fieldValues.get(i); + if (src != null) + LOGGER.error("filed value: " + new String(src)); + } + } + break; + default: + LOGGER.warn("Error connected status.", status); + break; + } + } + } + } + /** * * @return diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java new file mode 100644 index 0000000..7095773 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java @@ -0,0 +1,46 @@ +package io.mycat.sqlcache.impl.directmem; + + + +import io.mycat.sqlcache.IBufferPageFactory; +import io.mycat.sqlcache.MyCatBufferPage; + +import java.io.IOException; +import java.util.Set; + +/** + * Direct Memroy Page Factory + * + * @author zagnix + * @create 2016-12-28 07:46 + */ + +public class DirectMemoryFactory implements IBufferPageFactory { + public MyCatBufferPage acquirePage(long index) throws IOException { + return null; + } + + public void releasePage(long index) { + + } + + public void deletePage(long index) throws IOException { + + } + + public void flush() { + + } + + public void deleteAllPages() throws IOException { + + } + + public Set getExistingPageIndexSet() { + return null; + } + + public void releaseCachedPages() throws IOException { + + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java new file mode 100644 index 0000000..85dff63 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java @@ -0,0 +1,56 @@ +package io.mycat.sqlcache.impl.directmem; + + +import io.mycat.sqlcache.ISQLResult; + +import java.io.IOException; + +/** + * 基于Direct Memory Cache,用queue实现 + * + * @author zagnix + * @create 2016-12-28 07:46 + */ + +public class DirectMemorySQLResult implements ISQLResult { + public long put(byte[] data) throws IOException { + return 0; + } + + public byte[] next() throws IOException { + return new byte[0]; + } + + public byte[] get(long index) throws IOException { + return new byte[0]; + } + + public long size() { + return 0; + } + + public boolean isEmpty() { + return false; + } + + public boolean isFull() { + return false; + } + + public void flush() { + + } + + public void removeAll() throws IOException { + + } + + public void recycle() throws IOException { + + } + + @Override + public void reset() { + + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java new file mode 100644 index 0000000..46d81da --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java @@ -0,0 +1,307 @@ +package io.mycat.sqlcache.impl.mmap; + + + +import io.mycat.sqlcache.IBufferPageFactory; +import io.mycat.sqlcache.MyCatBufferPage; +import io.mycat.sqlcache.PageLRUCache; +import io.mycat.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static java.nio.channels.FileChannel.MapMode.READ_WRITE; + +/** + * Memroy Mapped Page Factory + * + * @author zagnix + * @create 2016-11-18 16:46 + */ + +public class MappedBufferPageFactory implements IBufferPageFactory { + private final static Logger logger = + LoggerFactory.getLogger(MappedBufferPageFactory.class); + private int pageSize; + private String pageDirName; + private File pageDir; + private String page; + + + private final Map pageLockMap = + new ConcurrentHashMap(); + + public static final String PAGE_FILE_NAME = "page"; + public static final String PAGE_FILE_SUFFIX = ".page"; + + private final PageLRUCache pageLRUCache; + private long cacheTTL = 0L; + + + public MappedBufferPageFactory(int pageFileSize, String pageDir, long cacheTTL){ + + /** + * 页文件大小 + */ + this.pageSize = pageFileSize; + /** + * 页文件目录名 + */ + this.pageDirName = pageDir; + /** + * 页文件目录 + */ + this.pageDir = new File(this.pageDirName); + this.cacheTTL = cacheTTL; + + + if (!this.pageDir.exists()) { + this.pageDir.mkdirs(); + } + + if (!this.pageDirName.endsWith(File.separator)) { + this.pageDirName += File.separator; + } + /** + * 页文件名前缀 + */ + this.page = this.pageDirName + PAGE_FILE_NAME + "-"; + this.pageLRUCache = new PageLRUCache(); + + } + + /** + * 查询page + * @param index + * @return + * @throws IOException + */ + public MyCatBufferPage acquirePage(long index) throws IOException { + /** + * 首先从Cache中查找对应的页 + */ + MyCatBufferPage myCatBufferPage = pageLRUCache.get(index); + + if (myCatBufferPage == null) { + try { + if (!pageLockMap.containsKey(index)) { + pageLockMap.put(index,new ReentrantReadWriteLock()); + } + Lock writeLock = pageLockMap.get(index).writeLock(); + /** + * 加锁处理当前index + */ + try { + writeLock.lock(); + /** + * 可能其他线程已经加入到了cache中了。 + */ + myCatBufferPage = pageLRUCache.get(index); + + if (myCatBufferPage == null) { + /** + * 创建一个页，利用Java MappedByteBuffer 高性能内存映射 + */ + RandomAccessFile raf = null; + FileChannel channel = null; + try { + String pageName = this.getPageNameByIndex(index); + raf = new RandomAccessFile(pageName, "rw"); + channel = raf.getChannel(); + + /** + * 文件映射 转换成 MappedByteBuffer + */ + MappedByteBuffer mbb = channel.map(READ_WRITE,0,this.pageSize); + + /** + * 转换成逻辑的映射页 + */ + myCatBufferPage = new MappedMemFileBufferPage(mbb, pageName, index,cacheTTL); + + /** + * 缓存起来。。。lRU Cache....... + */ + pageLRUCache.put(index, myCatBufferPage); + + if (logger.isDebugEnabled()) { + logger.debug("Mapped page for " + pageName + " was just created and cached."); + } + + } finally { + if (channel != null) channel.close(); + if (raf != null) raf.close(); + } + } + }finally { + writeLock.unlock(); + } + } finally { + pageLockMap.remove(index); + } + } + return myCatBufferPage; + } + + /** + * 根据索引生成页文件 + * @param index + * @return + */ + private String getPageNameByIndex(long index) { + return this.page + index + PAGE_FILE_SUFFIX; + } + + /** + * 回收Page + * @param index + */ + public void releasePage(long index) { + pageLRUCache.release(index); + } + + /** + * 通过page文件名,返回其page索引 + * @param pageName + * @return + */ + private long getIndexByPageName(String pageName) { + int beginIndex = pageName.lastIndexOf('-'); + beginIndex += 1; + int endIndex = pageName.lastIndexOf(PAGE_FILE_SUFFIX); + String sIndex = pageName.substring(beginIndex, endIndex); + long index = Long.parseLong(sIndex); + return index; + } + + + /** + * 根据索引删除页文件，调用者需要同步访问 + * @param index + * @throws IOException + */ + public void deletePage(long index) throws IOException { + pageLRUCache.remove(index); + String pageName = this.getPageNameByIndex(index); + int count = 0; + int maxRound = 10; + boolean deleted = false; + while(count < maxRound) { + try { + Utils.deleteFile(new File(pageName)); + deleted = true; + break; + } catch (IllegalStateException ex) { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + } + count++; + if (logger.isDebugEnabled()) { + logger.warn("fail to delete " + pageName + ", tried round = " + count); + } + } + } + if (deleted) { + if(logger.isDebugEnabled()) { + logger.debug("Page " + pageName + " was just deleted."); + } + } else { + logger.warn("fail to delete " + pageName + " after max " + maxRound + " rounds of try, you may delete it manually."); + } + } + + + /** + * thread unsafe, caller need synchronization + */ + public void flush() { + Collection cachedPages = pageLRUCache.getValues(); + for(MyCatBufferPage mappedPage : cachedPages) { + ((MappedMemFileBufferPage)mappedPage).flush(); + } + } + + /** + * thread unsafe, caller need synchronization + */ + public void deleteAllPages() throws IOException { + pageLRUCache.removeAll(); + Set indexSet = getExistingPageIndexSet(); + this.deletePages(indexSet); + } + + /** + * thread unsafe, caller need synchronization + */ + public void deletePages(Set indexes) throws IOException { + if (indexes == null) return; + for(long index : indexes) { + this.deletePage(index); + } + } + + public Set getExistingPageIndexSet() { + Set indexSet = new HashSet(); + File[] pageFiles = this.pageDir.listFiles(); + if (pageFiles != null && pageFiles.length > 0) { + for(File pageFile : pageFiles) { + String fileName = pageFile.getName(); + if (fileName.endsWith(PAGE_FILE_SUFFIX)) { + long index = this.getIndexByPageName(fileName); + indexSet.add(index); + } + } + } + return indexSet; + } + + public void releaseCachedPages() throws IOException { + pageLRUCache.removeAll(); + } + + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + + public String getPageDirName() { + return pageDirName; + } + + public void setPageDirName(String pageDirName) { + this.pageDirName = pageDirName; + } + + public File getPageDir() { + return pageDir; + } + + public void setPageDirFile(File pageDir) { + this.pageDir = pageDir; + } + + public String getPage() { + return page; + } + + public void setPageFile(String page) { + this.page = page; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java new file mode 100644 index 0000000..f63640b --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java @@ -0,0 +1,193 @@ +package io.mycat.sqlcache.impl.mmap; + + +import io.mycat.sqlcache.MyCatBufferPage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * 内存映射文件实现data存储 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-27 18:52 + */ + +public class MappedMemFileBufferPage extends MyCatBufferPage { + private final static Logger logger = + LoggerFactory.getLogger(MappedMemFileBufferPage.class); + private String pageName = null; + private long pageIndex = 0L; + private static Method mappedByteBufferCleaner = null; + private static Method mappedByteBufferCleanerClean = null; + + static { + try { + mappedByteBufferCleaner = + Class.forName("java.nio.DirectByteBuffer").getMethod("cleaner"); + mappedByteBufferCleaner.setAccessible(true); + mappedByteBufferCleanerClean = + Class.forName("sun.misc.Cleaner").getMethod("clean"); + mappedByteBufferCleanerClean.setAccessible(true); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + + /** + * 创建 一个MappedByteBufferPage对象 + * + * @param mappedByteBuffer + * @param file + * @param index + * @param cacheTTL + */ + public MappedMemFileBufferPage(MappedByteBuffer mappedByteBuffer, String file, long index, long cacheTTL){ + super(mappedByteBuffer.load(),cacheTTL); + this.pageName = file; + this.pageIndex = index; + } + + + /** + * + * @param position + * @param length + * @return + */ + public byte[] getBytes(int position, int length) { + ByteBuffer buf = this.getLocalByteBuffer(position); + byte[] data = new byte[length]; + buf.get(data); + return data; + } + + /** + * ByteBuffer slice from Thread Local + * @param position + * @param limit + * @return + */ + public ByteBuffer slice(int position, int limit) { + ByteBuffer buffer = this.threadLocalByteBuffer.get(); + buffer.limit(position+limit); + buffer.position(position); + return buffer.slice(); + } + + /** + * ByteBuffer from Thread Local + * + * @param position + * @return + */ + public ByteBuffer getLocalByteBuffer(int position) { + ByteBuffer buf = this.threadLocalByteBuffer.get(); + buf.position(position); + return buf; + } + + + /** + * 将pageName的数据刷入磁盘 + */ + public void flush() { + synchronized(this) { + if (this.recycled) return; + if (dirty) { + MappedByteBuffer mappedByteBuffer = + (MappedByteBuffer)threadLocalByteBuffer.getByteBuffer(); + mappedByteBuffer.force(); + dirty = false; + } + } + + } + + /** + * 返回 页 号 + * @return + */ + public long getPageIndex() { + return this.pageIndex; + } + + /** + * 设置 页 为 dirty + * @param dirty + */ + public void setDirty(boolean dirty) { + this.dirty = dirty; + } + + /** + * 回收页，关闭文件pageName + * @throws IOException + */ + public void recycle() throws IOException{ + + synchronized(this) { + if (this.recycled) return; + flush(); + MappedByteBuffer srcBuf = + (MappedByteBuffer)threadLocalByteBuffer.getByteBuffer(); + unmap(srcBuf); + this.threadLocalByteBuffer = null; + this.recycled = true; + } + + } + + /** + * 解文件内存映射 + * + * @param buffer + */ + private void unmap(final MappedByteBuffer buffer) { + if(buffer == null) + return; + + AccessController.doPrivileged(new PrivilegedAction() { + public Object run() { + try { + if (mappedByteBufferCleaner != null && + mappedByteBufferCleanerClean != null && + buffer.isDirect()) { + Object cleaner = mappedByteBufferCleaner.invoke(buffer); + mappedByteBufferCleanerClean.invoke(cleaner); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + return null; + } + }); + } + + + public boolean isRecycled() { + return false; + } + + public String getPageName() { + return pageName; + } + + public void setPageName(String pageName) { + this.pageName = pageName; + } + + public void setPageIndex(long pageIndex) { + this.pageIndex = pageIndex; + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java new file mode 100644 index 0000000..5f80355 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java @@ -0,0 +1,890 @@ +package io.mycat.sqlcache.impl.mmap; + + +import io.mycat.sqlcache.AddressIndex; +import io.mycat.sqlcache.ISQLResult; +import io.mycat.sqlcache.MyCatBufferPage; +import io.mycat.util.UnsafeMemory; +import io.mycat.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * 基于内存映射Cache,用queue实现 + * + * @author zagnix + * @create 2016-12-02 15:46 + */ +public class MappedSQLResult implements ISQLResult { + private final static Logger LOGGER = LoggerFactory.getLogger(MappedSQLResult.class); + /** + * 存放索引文件的目录 + */ + private static final String INDEX_PAGE_FOLDER = "index"; + /** + * 存放数据的目录 + */ + private static final String DATA_PAGE_FOLDER = "data"; + /** + * 保存元数据信息目录 + */ + private static final String META_DATA_PAGE_FOLDER = "meta_data"; + /** + * Page File Size + */ + private final int DATA_PAGE_SIZE; + + /** + * 默认页文件大小 + */ + public final static int DEFAULT_DATA_PAGE_SIZE = 128 * 1024 * 1024; + + /** + * 最小页文件大小 + */ + public final static int MINIMUM_DATA_PAGE_SIZE = 8 * 1024 * 1024; + + /** + * 索引页在内存中停留时间 + */ + public final static int INDEX_PAGE_CACHE_TTL = 1000; + /** + * 数据也在内存中停留时间 + */ + public final static int DATA_PAGE_CACHE_TTL = 1000; + /** + *索引文件地址编码,使用 64bit, + * 一条索引地址格式为： + * page number | page offset | data lenghe | op + * 63 50 | 49 21 | 20 1| 0 + */ + /** + * 最大data page 文件数16 * 1024 + */ + public final static int DATA_PAGE_NUMBER_BITS = 14; + /** + * 每个data文件大小最大为512M + */ + public final static int DATA_PAGE_INDEX_ITEM_OFFSET_BITS = 29; + /** + * 每次添加的数据最大长度1M + */ + public final static int DATA_ITEM_LEN_BITS = 20; + + /** + * 标志是否被消费了。用于多线程并发访问标志 + */ + public final static int DATA_PAGE_INDEX_OP_BITS = 1; + + /** + * 最大DATA页大小 + */ + public final static int MAX_DATA_PAGE_SIZE = 1 << DATA_PAGE_INDEX_ITEM_OFFSET_BITS; + /** + *最大DATA页的数量 + */ + public final static int MAX_DATA_PAGE_INDEX_SIZE = 1 << DATA_PAGE_NUMBER_BITS; + /** + * 每条item data最大长度 + */ + public final static int MAX_DATA_ITEM_LEN = 1 << DATA_ITEM_LEN_BITS; + /** + * 位操作用的 + */ + private static final long MASK_LONG_14_BITS = 0xFFF3L; + private static final long MASK_LONG_29_BITS = 0x1FFFFFFFL; + private static final long MASK_LONG_20_BITS = 0xFFFFFL; + private static final long MASK_LONG_1_BITS = 0x1L; + + /** + * 存放页文件目录 + */ + private String cacheDirectory = null; + + /** + * 索引页管理对象，提供acquire/release/cache + */ + private MappedBufferPageFactory indexPageFactory = null; + + /** + * 数据页管理对象，提供acquire/release/cache + */ + private MappedBufferPageFactory dataPageFactory = null; + + /** + * 元数据管理对象，提供acquire/release/cache + */ + private MappedBufferPageFactory metaPageFactory = null; + + private static final int META_DATA_PAGE_INDEX = 0; + private static final int META_DATA_ITEM_LENGTH_BITS = 4; + private static final int META_DATA_PAGE_SIZE = 1 << META_DATA_ITEM_LENGTH_BITS; + + /** + * index页存放data item数量是1024 * 1024条 + */ + private static final int INDEX_ITEMS_PER_PAGE_BITS = 20; + private static final int INDEX_ITEMS_PER_PAGE = 1 << INDEX_ITEMS_PER_PAGE_BITS; + /** + * 每条data item的编码 index address的大小为8bit + */ + private static final int INDEX_ITEM_LENGTH_BITS = 3; + private static final int INDEX_ITEM_LENGTH = 1 << INDEX_ITEM_LENGTH_BITS; + /** + * index 页文件大小为8*1024*1024 + */ + private static final int INDEX_PAGE_SIZE = INDEX_ITEM_LENGTH * INDEX_ITEMS_PER_PAGE; + + /** + * 队列头下标 + */ + private final AddressIndex queueFrontIndex = new AddressIndex(); + /** + * 队列尾下标 + */ + private final AddressIndex queueTailIndex = new AddressIndex(); + /** + * queueTailIndex下标 索引的data page + */ + private final AddressIndex queueTailDataPageIndex = new AddressIndex(); + + /** + * queueTailIndex下标索引的data page的当前写位置 + */ + private final AddressIndex queueTailDataItemOffset = new AddressIndex(); + + /** + * lock + */ + private final Lock appendLock = new ReentrantLock(); + private final ReadWriteLock arrayReadWritelock = new ReentrantReadWriteLock(); + private final Lock queueReadLock = arrayReadWritelock.readLock(); + private final Lock queueWriteLock = arrayReadWritelock.writeLock(); + + + + /** + * MappedBigCache构造函数 + * @param cacheDir + * @param cacheName + * @throws IOException + */ + public MappedSQLResult(String cacheDir, String cacheName) throws IOException{ + this(cacheDir,cacheName,DEFAULT_DATA_PAGE_SIZE); + } + + /** + * MappedSQLResult 构造函数 + * @param cacheDir + * @param cacheName + * @param cacheSize + * @throws IOException + */ + public MappedSQLResult(String cacheDir, String cacheName, int cacheSize) throws IOException { + + this.cacheDirectory = cacheDir; + /** + * 目录没有加文件分割符 + */ + if (!cacheDirectory.endsWith(File.separator)) { + cacheDirectory += File.separator; + } + + /** + * 构建一个消息队列名字 + */ + cacheDirectory = cacheDirectory + cacheName + File.separator; + + /** + * 确定路径合法 + */ + if (!Utils.isFilenameValid(cacheDirectory)) { + throw new IllegalArgumentException("invalid array directory : " + cacheDirectory); + } + + long pageSize = UnsafeMemory.roundToOsPageSzie((long)cacheSize); + + if (pageSize < MINIMUM_DATA_PAGE_SIZE) { + throw new IllegalArgumentException("invalid page size, allowed minimum is : " + MINIMUM_DATA_PAGE_SIZE + " bytes."); + } + + if (pageSize > MAX_DATA_PAGE_SIZE){ + throw new IllegalArgumentException("invalid page size, allowed max is : " + MAX_DATA_PAGE_SIZE + " bytes."); + } + + DATA_PAGE_SIZE = (int)pageSize; + this.init(); + } + + + private void init() throws IOException { + /** + * 索引页管理对象构建 + */ + this.indexPageFactory = new MappedBufferPageFactory(INDEX_PAGE_SIZE, + this.cacheDirectory + INDEX_PAGE_FOLDER, + INDEX_PAGE_CACHE_TTL); + + /** + * 数据页管理对象构建 + */ + this.dataPageFactory = new MappedBufferPageFactory(DATA_PAGE_SIZE, + this.cacheDirectory + DATA_PAGE_FOLDER, + DATA_PAGE_CACHE_TTL); + + /** + * 元信息管理对象构建 + */ + this.metaPageFactory = new MappedBufferPageFactory(META_DATA_PAGE_SIZE, + this.cacheDirectory + META_DATA_PAGE_FOLDER, + 10 * 1000); + + /** + * 初始化数组head 和 tail下标 + */ + initArrayIndex(); + + /** + * 初始化当前页号和偏移量 + */ + initDataPageIndex(); + + } + + + /** + * 初始化Queue的 front index 和 tail index + * @throws IOException + */ + void initArrayIndex() throws IOException { + /** + * 从元数据信息中获取数组的head / tail index + */ + MyCatBufferPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); + ByteBuffer metaBuf = metaDataPage.getLocalByteBuffer(0); + + long front= metaBuf.getLong(); + long tail = metaBuf.getLong(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(" head " + front); + LOGGER.debug(" tail " + tail); + } + /** + * head .tail 是保存在文件中， + * 程序启动时候，会读取数据元文件的head，tail信息初始化一个数组的head和tail + */ + queueFrontIndex.set(front); + queueTailIndex.set(tail); + } + + /** + * 初始化当前页号和偏移量 + * @throws IOException + */ + void initDataPageIndex() throws IOException { + /** + * 数组为空的情况 + */ + if (this.isEmpty()) { + queueTailDataPageIndex.set(0); + queueTailDataItemOffset.set(0); + } else { + + MyCatBufferPage previousIndexPage = null; + long previousIndexPageIndex = -1; + + try { + /** + * 根据数组的信息初始化Data Page的写位置 = tailIndex -1 + */ + long previousIndex = this.queueTailIndex.get() - 1; + + if (previousIndex < 0) { + previousIndex = Long.MAX_VALUE; + } + + /** + * previousIndex通过取模找到previousIndex所在的 index page + */ + previousIndexPageIndex = previousIndex / (INDEX_ITEMS_PER_PAGE); + + if (LOGGER.isDebugEnabled()) { + LOGGER.info("====>previousIndexPageIndex " + previousIndexPageIndex); + } + + /** + * MemoryMappedFile Index Page + */ + previousIndexPage = this.indexPageFactory.acquirePage(previousIndexPageIndex); + + /** + * 当前 index page内写位置 + */ + int previousIndexPageOffset = (int)(previousIndex * INDEX_ITEM_LENGTH); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("====>previousIndexPageOffset " + previousIndex); + } + + /** + * 根据页内偏移的位置得到对应的ByteBuffer + */ + ByteBuffer previousIndexItemBuffer = previousIndexPage.getLocalByteBuffer(previousIndexPageOffset); + + /** + * data index address + * data page number|data page offset|data len|op + */ + long previousDataPageIndexAddress = previousIndexItemBuffer.getLong(); + + /** + * 整个Data Page中当前的页号。 + * 以及Page内当前的偏移量。 + */ + queueTailDataPageIndex.set(decodeDataPageNumber(previousDataPageIndexAddress)); + queueTailDataItemOffset.set(decodeDataPageCurrentOffset(previousDataPageIndexAddress) + + decodeDataLen(previousDataPageIndexAddress)); + } finally { + /** + * 释放当前页 + */ + if (previousIndexPage != null) { + this.indexPageFactory.releasePage(previousIndexPageIndex); + } + } + } + } + /** + * 把data存到数据中 + * @param data + * @return + * @throws IOException + */ + public long put(byte[] data) throws IOException { + try { + queueWriteLock.lock(); + + /** + * 数据长度超过1MB的情况 + */ + if (data.length > MAX_DATA_ITEM_LEN){ + throw new IOException("data length larger :" + MAX_DATA_ITEM_LEN); + } + + /** + * 数据页 + */ + MyCatBufferPage toAppendDataPage = null; + + /** + * 索引页 + */ + MyCatBufferPage toAppendIndexPage = null; + + /** + * 追加的索引下标.... + */ + long toAppendIndexPageIndex = -1L; + + /** + * 追加数据索引下标.... + */ + long toAppendDataPageIndex = -1; + long toAppendArrayIndex = -1L; + + try { + /** + * 锁住当前，只允许一个线程访问 + */ + appendLock.lock(); // only one thread can append + + /** + * 数据是否达到最大空间极限 + */ + if (this.isFull()) { // end of the world check:) + throw new IOException("ring space of java long type used up, the end of the world!!!"); + } + /** + * 大于Data Page的大小,跳到下一页 + */ + if (this.queueTailDataItemOffset.get() + data.length > DATA_PAGE_SIZE) { + /** + * 整个索引下标超过了MAX_DATA_PAGE_INDEX_SIZE，回到下标为0 + */ + if (this.queueTailDataPageIndex.get() == (MAX_DATA_PAGE_INDEX_SIZE)) { + this.queueTailDataPageIndex.set(0L); + } else { + this.queueTailDataPageIndex.incrementAndGet(); + } + /** + * 新的页内偏移设置0。 + */ + this.queueTailDataItemOffset.set(0L); + } + + /** + * 还是在当前页中直接赋值。。。。。 + */ + toAppendDataPageIndex = this.queueTailDataPageIndex.get(); + int toAppendDataItemOffset = (int)this.queueTailDataItemOffset.get(); + + /** + * 队列尾下标 + */ + toAppendArrayIndex = this.queueTailIndex.get(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("toAppendArrayIndex " + toAppendArrayIndex); + } + + /** + * 查找toAppendDataPageIndex对应的MemoryMappedFile对象 + */ + toAppendDataPage = this.dataPageFactory.acquirePage(toAppendDataPageIndex); + + /** + * 根据toAppendDataItemOffset此时的读写偏移位置返回一个ByteBuffer对象，并设置position + * 的位置为 toAppendDataItemOffset + */ + ByteBuffer toAppendDataPageBuffer = toAppendDataPage.getLocalByteBuffer(toAppendDataItemOffset); + + /** + * 将item的数据写入ByteBuffer中 + */ + toAppendDataPageBuffer.put(data); + + /** + * 当前页有数据了，标志为脏页 + */ + toAppendDataPage.setDirty(true); + + /** + * 更新当前page页面的数据item偏移位置。 + */ + this.queueTailDataItemOffset.addAndGet(data.length); + + /** + * 将数据页号和当前数据数据页号编码成 index address存放到索引页中 + */ + + toAppendIndexPageIndex = toAppendArrayIndex / (INDEX_ITEMS_PER_PAGE); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("index page number: " + toAppendIndexPageIndex); + } + + toAppendIndexPage = this.indexPageFactory.acquirePage(toAppendIndexPageIndex); + int toAppendIndexItemOffset = (int) ((toAppendArrayIndex&(INDEX_ITEMS_PER_PAGE-1))*INDEX_ITEM_LENGTH); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("index page offset: " + toAppendIndexItemOffset); + } + + ByteBuffer toAppendIndexPageBuffer = toAppendIndexPage.getLocalByteBuffer(toAppendIndexItemOffset); + + /** + * 就是记录data item 所在的数据页号，数据页偏移位置，item数据长度 + */ + toAppendIndexPageBuffer.putLong(encodeIndexAddress((int)toAppendDataPageIndex,toAppendDataItemOffset,data.length,(byte) 0x1)); + toAppendIndexPage.setDirty(true); + /** + * 将数据的头加1 + */ + this.queueTailIndex.incrementAndGet(); + + /** + * 更新元数据,只有一页的0 + * 保存当前的数组的头和尾信息 + */ + MyCatBufferPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); + ByteBuffer metaDataBuf = metaDataPage.getLocalByteBuffer(0); + metaDataBuf.putLong(this.queueFrontIndex.get()); + metaDataBuf.putLong(this.queueTailIndex.get()); + metaDataPage.setDirty(true); + + } finally { + + appendLock.unlock(); + + if (toAppendDataPage != null) { + this.dataPageFactory.releasePage(toAppendDataPageIndex); + } + if (toAppendIndexPage != null) { + this.indexPageFactory.releasePage(toAppendIndexPageIndex); + } + } + + return toAppendArrayIndex; + + } finally { + queueWriteLock.unlock(); + } + } + + /** + * 取数据 + * @return + * @throws IOException + */ + public byte[] next() throws IOException { + try { + queueWriteLock.lock(); + + if (this.isEmpty()) { + return null; + } + + long frontIndex = this.queueFrontIndex.get(); + + + MyCatBufferPage dataPage = null; + long dataItemIndexAddress = -1L; + int dataPageNumber = -1; + try { + /** + * 根据index，获取index address。 + */ + ByteBuffer indexItemBuffer = this.getIndexItemBuffer(frontIndex); + + /** + * 得到data item 的索引地址，通过索引地址解码出页号，页偏移，data Item 的长度 + */ + dataItemIndexAddress = indexItemBuffer.getLong(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("dataItemIndexAddress " + dataItemIndexAddress); + } + + dataPageNumber = decodeDataPageNumber(dataItemIndexAddress); + int dataItemOffset = decodeDataPageCurrentOffset(dataItemIndexAddress); + int dataItemLength = decodeDataLen(dataItemIndexAddress); + + dataPage = this.dataPageFactory.acquirePage(dataPageNumber); + + byte[] data = dataPage.getBytes(dataItemOffset,dataItemLength); + + long nextQueueFrontIndex = frontIndex; + if (nextQueueFrontIndex == Long.MAX_VALUE) { + nextQueueFrontIndex = 0L; + } else { + nextQueueFrontIndex++; + } + this.queueFrontIndex.set(nextQueueFrontIndex); + MyCatBufferPage queueFrontIndexPage = + this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); + ByteBuffer queueFrontIndexBuffer = + queueFrontIndexPage.getLocalByteBuffer(0); + queueFrontIndexBuffer.putLong(queueFrontIndex.get()); + queueFrontIndexBuffer.putLong(queueTailIndex.get()); + queueFrontIndexPage.setDirty(true); + return data; + } finally { + if (dataPage != null) { + this.dataPageFactory.releasePage(dataPageNumber); + } + } + } finally { + queueWriteLock.unlock(); + } + } + + + /** + *get下标为index的数据 + * @param index + * @return + * @throws IOException + */ + public byte[] get(long index) throws IOException { + try { + queueReadLock.lock(); + validateIndex(index); + MyCatBufferPage dataPage = null; + long dataItemIndexAddress = -1L; + int dataPageNumber = -1; + try { + + /** + * 根据index，获取index address。 + */ + ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); + + /** + * 得到data item 的索引地址，通过索引地址解码出页号，页偏移，data Item 的长度 + */ + dataItemIndexAddress = indexItemBuffer.getLong(); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("dataItemIndexAddress " + dataItemIndexAddress); + } + + dataPageNumber = decodeDataPageNumber(dataItemIndexAddress); + int dataItemOffset = decodeDataPageCurrentOffset(dataItemIndexAddress); + int dataItemLength = decodeDataLen(dataItemIndexAddress); + + dataPage = this.dataPageFactory.acquirePage(dataPageNumber); + + byte[] data = dataPage.getBytes(dataItemOffset,dataItemLength); + + return data; + } finally { + if (dataPage != null) { + this.dataPageFactory.releasePage(dataPageNumber); + } + } + } finally { + queueReadLock.unlock(); + } + } + + /** + * 队列大小 + * @return + */ + public long size() { + long qFront = this.queueFrontIndex.get(); + long qRear = this.queueTailIndex.get(); + if (qFront <= qRear) { + return (qRear - qFront); + } else { + return Long.MAX_VALUE - qFront + 1 + qRear; + } + } + + /** + * 确定index是否合法 + * @param index + */ + + void validateIndex(long index) { + if (this.queueFrontIndex.get() <= this.queueTailIndex.get()) { + if (index < this.queueFrontIndex.get() || index >= this.queueTailIndex.get()) { + throw new IndexOutOfBoundsException(); + } + } else { + if (index < this.queueFrontIndex.get() && index >= this.queueTailIndex.get()) { + throw new IndexOutOfBoundsException(); + } + } + } + + /** + * 从索引文件中找到数组下标index对应的索引ByteBuffer + * @param index + * @return + * @throws IOException + */ + ByteBuffer getIndexItemBuffer(long index) throws IOException { + + MyCatBufferPage indexPage = null; + long indexPageIndex = -1L; + try { + + indexPageIndex = index / (INDEX_ITEMS_PER_PAGE); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("get index " + indexPageIndex); + } + + indexPage = this.indexPageFactory.acquirePage(indexPageIndex); + int indexItemOffset = (int) ((index&(INDEX_ITEMS_PER_PAGE-1)) * INDEX_ITEM_LENGTH); + ByteBuffer indexItemBuffer = indexPage.getLocalByteBuffer(indexItemOffset); + return indexItemBuffer; + + } finally { + if (indexPage != null) { + this.indexPageFactory.releasePage(indexPageIndex); + } + } + } + + /** + * 将Data页号，以及当前可写偏移位置，写入data的长度，以及op位编码成一个64为的地址 + * @param dataPageNumber + * @param currentOffset + * @param dataLen + * @param op + * @return + */ + public static long encodeIndexAddress(int dataPageNumber, + int currentOffset,int dataLen,byte op) { + + return ((long)dataPageNumber)<<(DATA_PAGE_INDEX_ITEM_OFFSET_BITS+DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS)| + ((long)currentOffset)<<(DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS)| + ((long)dataLen)<>> + (DATA_PAGE_INDEX_ITEM_OFFSET_BITS + + DATA_ITEM_LEN_BITS+ + DATA_PAGE_INDEX_OP_BITS))&MASK_LONG_14_BITS); + } + + /** + * 从索引地址中分离出当前data页写的位置 + * @param indexAddress + * @return + */ + private static int decodeDataPageCurrentOffset(long indexAddress) { + return (int)((indexAddress >>> (DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS))&MASK_LONG_29_BITS); + } + + /** + * 从索引地址分离出当前data页的存放数据长度 + * @param indexAddress + * @return + */ + private static int decodeDataLen(long indexAddress) { + return (int)((indexAddress >>> DATA_PAGE_INDEX_OP_BITS)&MASK_LONG_20_BITS); + } + + /** + * 从索引地址分离出最低位的值 + * @param indexAddress + * @return + */ + private static byte decodeOpBit(long indexAddress) { + return (byte) (indexAddress&MASK_LONG_1_BITS); + } + + + /** + * 判断队列是否为空？ + * @return + */ + public boolean isEmpty() { + try { + queueReadLock.lock(); + return this.queueFrontIndex.get() == this.queueTailIndex.get(); + } finally { + queueReadLock.unlock(); + } + } + + /** + * 判断队列是否满了 + * @return + */ + public boolean isFull() { + try { + queueReadLock.lock(); + long currentIndex = this.queueTailIndex.get(); + long nextIndex = currentIndex == (Long.MAX_VALUE) ? 0 : currentIndex + 1; + return nextIndex == this.queueFrontIndex.get(); + } finally { + queueReadLock.unlock(); + } + } + + + /** + * 队头下标 + * @return + */ + public long getQueueFrontIndex() { + try { + queueReadLock.lock(); + return queueFrontIndex.get(); + } finally { + queueReadLock.unlock(); + } + } + + /** + * 队尾下标 + * @return + */ + public long getQueueTailIndex() { + try { + queueReadLock.lock(); + return queueTailIndex.get(); + } finally { + queueReadLock.unlock(); + } + } + + + + /** + * 刷页面到磁盘总 + */ + public void flush() { + try { + queueReadLock.lock(); + this.metaPageFactory.flush(); + this.indexPageFactory.flush(); + this.dataPageFactory.flush(); + } finally { + queueReadLock.unlock(); + } + } + + /** + * 从cache中移走，并unmap操作 + * @throws IOException + */ + public void recycle() throws IOException { + try { + queueWriteLock.lock(); + if (this.metaPageFactory != null) { + this.metaPageFactory.releaseCachedPages(); + } + if (this.indexPageFactory != null) { + this.indexPageFactory.releaseCachedPages(); + } + if (this.dataPageFactory != null) { + this.dataPageFactory.releaseCachedPages(); + } + } finally { + queueWriteLock.unlock(); + } + } + + /** + * reset front index = 0 + */ + public void reset(){ + this.queueFrontIndex.set(0); + MyCatBufferPage metaDataPage = null; + try { + metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); + } catch (IOException e) { + e.printStackTrace(); + } + ByteBuffer metaDataBuf = metaDataPage.getLocalByteBuffer(0); + metaDataBuf.putLong(this.queueFrontIndex.get()); + metaDataBuf.putLong(this.queueTailIndex.get()); + metaDataPage.setDirty(true); + } + + /** + * 删除所有文件并初始化环境 + * @throws IOException + */ + public void removeAll() throws IOException { + try { + queueWriteLock.lock(); + this.indexPageFactory.deleteAllPages(); + this.dataPageFactory.deleteAllPages(); + this.metaPageFactory.deleteAllPages(); + this.init(); + } finally { + queueWriteLock.unlock(); + } + } +} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java new file mode 100644 index 0000000..5fee33d --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java @@ -0,0 +1,7 @@ +package io.mycat.sqlcache; + +/** + * + * testCase: + * \test\java\io\mycat\bigmem\sqlcache\testBigSQLResultCache.java + * */ \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/util/UnsafeMemory.java b/Mycat-Core/src/main/java/io/mycat/util/UnsafeMemory.java new file mode 100644 index 0000000..d597df1 --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/util/UnsafeMemory.java @@ -0,0 +1,52 @@ +package io.mycat.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sun.misc.Unsafe; + +import java.lang.reflect.Field; + + +/** + * Unsafe 工具类 + * + * @author zagnix + * @create 2016-11-18 14:17 + */ +public final class UnsafeMemory { + private final static Logger logger = LoggerFactory.getLogger(UnsafeMemory.class); + private static final Unsafe _UNSAFE; + public static final int BYTE_ARRAY_OFFSET; + static { + Unsafe unsafe; + try { + Field theUnsafeField = Unsafe.class.getDeclaredField("theUnsafe"); + theUnsafeField.setAccessible(true); + unsafe = (Unsafe) theUnsafeField.get(null); + } catch (Exception e) { + logger.error(e.getMessage()); + unsafe = null; + } + _UNSAFE = unsafe; + if (_UNSAFE != null) { + BYTE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(byte[].class); + } else { + BYTE_ARRAY_OFFSET = 0; + } + } + public static Unsafe getUnsafe() { + return _UNSAFE; + } + + /** + * 将size规整化为pagesize的倍数 + * @param size + * @return + */ + public static long roundToOsPageSzie(long size) { + long pagesize = _UNSAFE.pageSize(); + return (size + (pagesize-1)) & ~(pagesize-1); + + } + +} diff --git a/Mycat-Core/src/main/java/io/mycat/util/Utils.java b/Mycat-Core/src/main/java/io/mycat/util/Utils.java new file mode 100644 index 0000000..70d4eab --- /dev/null +++ b/Mycat-Core/src/main/java/io/mycat/util/Utils.java @@ -0,0 +1,74 @@ +package io.mycat.util; + +import java.io.File; +import java.io.IOException; +import java.util.Random; +import java.util.regex.Pattern; + +public class Utils { + private static final String illegalChars = "/" + '\u0000' + '\u0001' + "-" + '\u001F' + '\u007F' + "-" + '\u009F' + '\uD800' + "-" + '\uF8FF' + '\uFFF0' + + "-" + '\uFFFF'; + private static final Pattern p = Pattern.compile("(^\\.{1,2}$)|[" + illegalChars + "]"); + + public static void validateFolder(String name) { + if (name == null || name.length() == 0) { + throw new IllegalArgumentException("folder name is emtpy"); + } + if(name.length() > 255) { + throw new IllegalArgumentException("folder name is too long"); + } + if (p.matcher(name).find()) { + throw new IllegalArgumentException("folder name [" + name + "] is illegal"); + } + } + + public static boolean isFilenameValid(String file) { + File f = new File(file); + try { + f.getCanonicalPath(); + return true; + } catch (IOException e) { + return false; + } + } + + public static void deleteDirectory(File dir) { + if (!dir.exists()) return; + File[] subs = dir.listFiles(); + if (subs != null) { + for (File f : dir.listFiles()) { + if (f.isFile()) { + if(!f.delete()) { + throw new IllegalStateException("delete file failed: "+f); + } + } else { + deleteDirectory(f); + } + } + } + if(!dir.delete()) { + throw new IllegalStateException("delete directory failed: "+dir); + } + } + + static final String AB = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + static Random rnd = new Random(); + + public static String randomString(int len ) + { + StringBuilder sb = new StringBuilder( len ); + for( int i = 0; i < len; i++ ) + sb.append( AB.charAt( rnd.nextInt(AB.length()) ) ); + return sb.toString(); + } + + + public static void deleteFile(File file) { + if (!file.exists() || !file.isFile()) { + return; + } + if (!file.delete()) { + throw new IllegalStateException("delete file failed: "+file); + } + } +} diff --git a/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java b/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java new file mode 100644 index 0000000..f1c87e6 --- /dev/null +++ b/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java @@ -0,0 +1,231 @@ +package io.mycat.sqlcache; + +import com.google.common.util.concurrent.*; +import io.mycat.util.Utils; + + +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; + +import static com.google.common.hash.Hashing.murmur3_32; + +/** + * Cache SQL 大结果集 对外接口 + * + * @author zagnix + * @version 1.0 + * @create 2016-12-30 10:51 + */ + +public class MyCatBigSqlResultsCache { + + private final CacheImp sqlResultCacheImp; + + private final static MyCatBigSqlResultsCache INSTANCE = new MyCatBigSqlResultsCache(); + + private MyCatBigSqlResultsCache(){ + sqlResultCacheImp = new CacheImp(); + } + + /** + * 将sql结果集缓存起来 + * + * @param sql sql语句 + * @param bigSQLResult sql结果集缓存 + * @param cache 缓存时间 + * @param accesCount 在缓存时间内，被访问的次数 + * @param loader 结果集load or reload 接口 + * @param listener key被移除时，调用的接口 + */ + public void cacheSQLResult(String sql, BigSQLResult bigSQLResult, long cache, long accesCount, + IDataLoader loader, IRemoveKeyListener listener){ + /** + * TODO + */ + String key = "" + murmur3_32().hashUnencodedChars(sql); + + Keyer keyer = new Keyer(); + keyer.setSql(sql); + keyer.setKey(key); + keyer.setValue(bigSQLResult); + keyer.setCacheTTL(cache); + keyer.setAccessCount(accesCount); + keyer.setRemoveKeyListener(listener); + keyer.setiDataLoader(loader); + sqlResultCacheImp.put(key,bigSQLResult,keyer); + } + + + /** + * 获取sql语句，已经缓存的结果集 + * + * @param sql sql 语句 + * @return + */ + public BigSQLResult getSQLResult(String sql){ + /** + * TODO + */ + String key = "" + murmur3_32().hashUnencodedChars(sql); + return sqlResultCacheImp.get(key); + } + + + + /** + * 获取sql语句，已经缓存的结果集 + * + * @param sql sql 语句 + */ + public void remove(String sql){ + /** + * TODO + */ + String key = "" + murmur3_32().hashUnencodedChars(sql); + sqlResultCacheImp.remove(key); + } + + + + /** + * 对外对象实例 + * @return + */ + public static MyCatBigSqlResultsCache getInstance() { + return INSTANCE; + } + + + public static void main(String [] args){ + + String sql = "select * from table"; + + BigSQLResult sqlResultCache = + new BigSQLResult(LocatePolicy.Normal,sql,32*1024*1024); + + long ROWS = 10000; + + + /** 模拟从后端DB拉取数据 */ + for (int i = 0; i < ROWS ; i++) { + byte[] rows = Utils.randomString(1024).getBytes(); + + /** + * 使用内存映射Cache，存放SQL结果集 + */ + sqlResultCache.put(rows); + } + + + MyCatBigSqlResultsCache.getInstance().cacheSQLResult(sql,sqlResultCache,300,5000, + new IDataLoader() { + + /** + * 根据sql,异步从后台DB reload数据，替换旧值 + * @param keyer + * @return + */ + @Override + public BigSQLResult reload(Keyer keyer) { + ListeningExecutorService executor = + MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()); + + final ListenableFuture listenableFuture = executor + .submit(new Callable() { + @Override + public BigSQLResult call() throws Exception { + String sql = keyer.getSql(); + String sqlkey = keyer.getLastAccessTime() +"_"+murmur3_32().hashUnencodedChars(sql); + BigSQLResult sqlResultCache + = new BigSQLResult(LocatePolicy.Normal,sqlkey,32*1024*1024); + + /**模拟从后端DB拉取数据*/ + for (int i = 0; i < ROWS ; i++) { + byte[] rows = Utils.randomString(1024).getBytes(); + + /** + * 使用内存映射Cache，存放SQL结果集 + */ + sqlResultCache.put(rows); + } + + return sqlResultCache; + } + }); + + Futures.addCallback(listenableFuture, new FutureCallback() { + @Override + public void onSuccess(BigSQLResult bigSQLResult) { + if(bigSQLResult!=null && bigSQLResult.hasNext()){ + + BigSQLResult oldSqlResult= + MyCatBigSqlResultsCache.getInstance().getSQLResult(keyer.getSql()); + + if (oldSqlResult != null){ + oldSqlResult.removeAll(); + } + + /**替换旧的值*/ + MyCatBigSqlResultsCache.getInstance(). + sqlResultCacheImp.put(keyer.getKey(),bigSQLResult,keyer); + } + } + + @Override + public void onFailure(Throwable t) { + //TODO + } + } + ); + + return null; + } + + + }, new IRemoveKeyListener() { + + /** + * key 失效，做清理工作 + * + * @param key + * @param value + */ + @Override + public void removeNotify(String key, BigSQLResult value) { + if (value !=null){ + value.removeAll(); + } + + System.out.println("key :" + key); + } + }); + + /** + * 访问Cache + */ + BigSQLResult bigSQLResult = + MyCatBigSqlResultsCache.getInstance().getSQLResult(sql); + + + bigSQLResult.reset(); + + while (bigSQLResult.hasNext()){ + + byte[] data = bigSQLResult.next(); + //TODO + System.out.println("String :" + new String(data)); + + } + + + MyCatBigSqlResultsCache.getInstance().remove(sql); + + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + +} diff --git a/Mycat-Core/src/test/java/io/mycat/sqlcache/TestHintSqlParser.java b/Mycat-Core/src/test/java/io/mycat/sqlcache/TestHintSqlParser.java new file mode 100644 index 0000000..a08195c --- /dev/null +++ b/Mycat-Core/src/test/java/io/mycat/sqlcache/TestHintSqlParser.java @@ -0,0 +1,17 @@ +package io.mycat.sqlcache; + +import org.junit.Test; + +/** + * @author zagnix + * @create 2017-01-16 18:03 + */ + +public class TestHintSqlParser { + //TODO 测试case待完善 + @Test + public void testHintSqlParser(){ + String hintSql = "/*!mycat:cacheable=true cache-time=5000 auto-refresh=true access-count=5000*/select * from table"; + System.out.println(HintSQLParser.parserHintSQL(hintSql).toString()); + } +} diff --git a/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java b/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java new file mode 100644 index 0000000..c0d4263 --- /dev/null +++ b/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java @@ -0,0 +1,128 @@ +package io.mycat.sqlcache; + + +import io.mycat.util.Utils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static com.google.common.hash.Hashing.murmur3_32; +import static org.junit.Assert.*; + +//TODO 待完善测试用例 +public class testSQLResultCache { + private BigSQLResult sqlResultCache; + + @Test + public void simpleTest() throws IOException { + for(int i = 1; i <= 2; i++) { + + sqlResultCache = new BigSQLResult(LocatePolicy.Normal, "select * from t",16*1024*1024); + assertNotNull(sqlResultCache); + + for(int j = 1; j <= 3; j++) { + assertTrue(sqlResultCache.size() == 0L); + assertTrue(sqlResultCache.isEmpty()); + + assertNull(sqlResultCache.next()); + + sqlResultCache.put("hello".getBytes()); + assertTrue(sqlResultCache.size() == 1L); + assertTrue(sqlResultCache.hasNext()); + assertEquals("hello", new String(sqlResultCache.next())); + assertNull(sqlResultCache.next()); + + sqlResultCache.put("world".getBytes()); + sqlResultCache.flush(); + assertTrue(sqlResultCache.size() == 1L); + assertTrue(sqlResultCache.hasNext()); + assertEquals("world", new String(sqlResultCache.next())); + assertNull(sqlResultCache.next()); + } + sqlResultCache.recycle(); + } + } + + + @Test + public void testSQLResultCache(){ + long ROWS = 10000; + //long ROWS = 10000*10000; + //long ROWS = 100000*100000; + Map sqlResultCacheMap = new HashMap(); + + String sql = "select * from table1"; + String sqlkey = ""+murmur3_32().hashUnencodedChars(sql); + + /** + * sql results back list + */ + + ArrayList backList = new ArrayList(); + + + /** + * 使用内存映射Cache，存放SQL结果集 + */ + + BigSQLResult sqlResultCache + = new BigSQLResult(LocatePolicy.Normal,sqlkey,16*1024*1024); + + for (int i = 0; i < ROWS ; i++) { + byte[] rows = Utils.randomString(1024).getBytes(); + backList.add(rows); + + /** + * 使用内存映射Cache，存放SQL结果集 + */ + sqlResultCache.put(rows); + } + sqlResultCacheMap.put(sqlkey,sqlResultCache); + + + + /** + * 验证内存映射Cache，存放SQL结果集 + */ + BigSQLResult sqlResCache = sqlResultCacheMap.get(sqlkey); + + Assert.assertEquals(backList.size(),sqlResCache.size()); + for (int i = 0; i Date: Thu, 16 Feb 2017 18:26:14 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E9=80=9A=E8=BF=87pom=E5=BC=95=E7=94=A8bigm?= =?UTF-8?q?emory=E9=A1=B9=E7=9B=AE=E5=AE=9E=E7=8E=B0sql=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E9=9B=86=E6=9C=AC=E5=9C=B0=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mycat-Core/pom.xml | 5 + .../callback/SQLResCacheHintHandler.java | 3 + .../java/io/mycat/sqlcache/AddressIndex.java | 94 -- .../java/io/mycat/sqlcache/BigSQLResult.java | 136 --- .../main/java/io/mycat/sqlcache/CacheImp.java | 175 ---- .../io/mycat/sqlcache/HintSQLDataLoader.java | 7 +- .../sqlcache/HintSQLRemoveKeyListener.java | 5 +- .../java/io/mycat/sqlcache/IBufferPage.java | 76 -- .../io/mycat/sqlcache/IBufferPageFactory.java | 63 -- .../main/java/io/mycat/sqlcache/ICache.java | 42 - .../java/io/mycat/sqlcache/IDataLoader.java | 18 - .../io/mycat/sqlcache/IRemoveKeyListener.java | 19 - .../java/io/mycat/sqlcache/ISQLResult.java | 76 -- .../main/java/io/mycat/sqlcache/Keyer.java | 113 --- .../io/mycat/sqlcache/MyCatBufferPage.java | 141 --- .../java/io/mycat/sqlcache/PageLRUCache.java | 231 ----- .../sqlcache/SQLResultsCacheService.java | 4 +- .../impl/directmem/DirectMemoryFactory.java | 46 - .../impl/directmem/DirectMemorySQLResult.java | 56 -- .../impl/mmap/MappedBufferPageFactory.java | 307 ------ .../impl/mmap/MappedMemFileBufferPage.java | 193 ---- .../sqlcache/impl/mmap/MappedSQLResult.java | 890 ------------------ .../java/io/mycat/sqlcache/package-info.java | 7 - 23 files changed, 20 insertions(+), 2687 deletions(-) delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java delete mode 100644 Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java diff --git a/Mycat-Core/pom.xml b/Mycat-Core/pom.xml index b05f3d1..acbcb7c 100644 --- a/Mycat-Core/pom.xml +++ b/Mycat-Core/pom.xml @@ -86,6 +86,11 @@ 21.0 + + io.mycat.bigmem + mycat-bigmemory + 0.0.1-RELEASE + diff --git a/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java index 8ec5985..7a8aadf 100644 --- a/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java +++ b/Mycat-Core/src/main/java/io/mycat/backend/callback/SQLResCacheHintHandler.java @@ -2,6 +2,9 @@ import io.mycat.backend.BackConnectionCallback; import io.mycat.backend.MySQLBackendConnection; +import io.mycat.bigmem.sqlcache.BigSQLResult; +import io.mycat.bigmem.sqlcache.IDataLoader; +import io.mycat.bigmem.sqlcache.IRemoveKeyListener; import io.mycat.front.MySQLFrontConnection; import io.mycat.mysql.packet.MySQLPacket; import io.mycat.net2.ConDataBuffer; diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java deleted file mode 100644 index c3d418e..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/AddressIndex.java +++ /dev/null @@ -1,94 +0,0 @@ -package io.mycat.sqlcache; - -import io.mycat.util.UnsafeMemory; -import sun.misc.Unsafe; - -/** - * cache padding from disruptor - * - * @author zagnix - * @create 2016-11-18 10:17 - */ - -class LhsPadding -{ - protected long p1, p2, p3, p4, p5, p6, p7; -} - -class Value extends LhsPadding -{ - protected volatile long value; -} - -class RhsPadding extends Value -{ - protected long p9, p10, p11, p12, p13, p14, p15; -} - -public class AddressIndex extends RhsPadding { - static final long INITIAL_VALUE = -1L; - private static final Unsafe UNSAFE = UnsafeMemory.getUnsafe(); - private static final long VALUE_OFFSET; - - public AddressIndex() { - this(INITIAL_VALUE); - } - - public AddressIndex(long initialValue) { - UNSAFE.putOrderedLong(this, VALUE_OFFSET, initialValue); - } - - public long get() { - return this.value; - } - - public void set(long value) { - UNSAFE.putOrderedLong(this, VALUE_OFFSET, value); - } - - public void setVolatile(long value) { - UNSAFE.putLongVolatile(this, VALUE_OFFSET, value); - } - - /** - * 内存地址值(通旧值this所在对象加上字节偏量VALUE_OFFSET找到所在的内存段) - * 期望值: expectedValue - * 新值: newValue - * 如果 内存地址值 == 期望值 - * 则将新的值写入到内存地址中。 - * return true - * else - * return false - * @param expectedValue - * @param newValue - * @return - */ - public boolean compareAndSet(long expectedValue, long newValue) { - return UNSAFE.compareAndSwapLong(this, VALUE_OFFSET, expectedValue, newValue); - } - - - public long incrementAndGet() { - return this.addAndGet(1L); - } - - public long addAndGet(long increment) { - long currentValue; - long newValue; - do { - currentValue = this.get(); - newValue = currentValue + increment; - } while(!this.compareAndSet(currentValue,newValue)); - return newValue; - } - public String toString() { - return Long.toString(this.get()); - } - static { - try { - VALUE_OFFSET = UNSAFE.objectFieldOffset(Value.class.getDeclaredField("value")); - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} \ No newline at end of file diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java deleted file mode 100644 index d8f0cba..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/BigSQLResult.java +++ /dev/null @@ -1,136 +0,0 @@ -package io.mycat.sqlcache; - - -import io.mycat.sqlcache.impl.mmap.MappedSQLResult; - -import java.io.IOException; -import java.util.Iterator; - -import static com.google.common.hash.Hashing.murmur3_32; - -/** - * SQL大结果集缓存 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-27 18:48 - */ - -public class BigSQLResult implements Iterator { - - private ISQLResult sqlResult; - private String cacheDir; - - public BigSQLResult(LocatePolicy locatePolicy, String sql, int pageSize){ - /** - * Core - DirectMemory - * Normal - mmap - */ - this.cacheDir = "sqlcache/"; - String sqlkey = ""+murmur3_32().hashUnencodedChars(sql); - - if (locatePolicy.equals(LocatePolicy.Normal)){ - try { - sqlResult = new MappedSQLResult(cacheDir,sqlkey,pageSize); - } catch (IOException e) { - e.printStackTrace(); - } - }else if(locatePolicy.equals(LocatePolicy.Core)){ - - } - } - - /** - * 添加一条sql二进制数据到Cache存储中 - * @param data - */ - public void put(byte [] data){ - try { - sqlResult.put(data); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * 数据是否为空 - * @return - */ - public boolean isEmpty(){ - return sqlResult.isEmpty(); - } - - /** - * 还有下一条sql结果集？ - * @return - */ - public boolean hasNext() { - return !sqlResult.isEmpty(); - } - - /** - * 取下一条sql结果集 - * - * @return - */ - public byte[] next() { - try { - return sqlResult.next(); - } catch (IOException e) { - e.printStackTrace(); - } - return null; - } - - /** - * Iterator interface - */ - public void remove() { - //TODO - } - - /** - * 主动将内存映射文件的数据刷到磁盘中 - */ - public void flush(){ - sqlResult.flush(); - } - - /** - * sql 结果集大小 - * @return - */ - public long size(){ - return sqlResult.size(); - } - - /** - * 从 PageLRUCache中移除Page，并执行unmap操作，并删除对于文件 - */ - public void removeAll(){ - try { - sqlResult.removeAll(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * 从 PageLRUCache中移除Page，并执行unmap操作，但不删除文件 - * 下次运行时候可以读取文件内容 - */ - public void recycle(){ - try { - sqlResult.recycle(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * 复位读位置为0，从头开始读取sql结果集 - */ - public void reset(){ - sqlResult.reset(); - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java deleted file mode 100644 index 2d8f781..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/CacheImp.java +++ /dev/null @@ -1,175 +0,0 @@ -package io.mycat.sqlcache; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.*; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - - -/** - * ICache实现类 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-30 16:53 - */ - -public class CacheImp implements ICache { - - private final static Logger logger - = LoggerFactory.getLogger(CacheImp.class); - - public static final long DEFAULT_TTL = 5 * 1000; - public static final int ACCESS_COUNT = 500; - private static final int REMOVE_ACTION = 1; - private static final int RELOAD_ACTION = 2; - - private final Map cacheMap; - private final Map> cacheKeyers; - - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private final Lock readLock = lock.readLock(); - private final Lock writeLock = lock.writeLock(); - - - /** - * 异步线程，按照cache规则生效 - * 1.缓存时间到期，进行del - * 2.如果在缓存时间内，访问次数超过N次，重新从DB后台load数据，更新Value - */ - private static final ThreadFactory threadFactory = - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("async-op-cache ").build(); - private static final ScheduledExecutorService scheduleAtFixedRate = - new ScheduledThreadPoolExecutor(1,threadFactory); - private final Set> keysToDel = new HashSet>(); - private final Set> keysToReload= new HashSet>(); - - - public CacheImp(){ - cacheMap = new ConcurrentHashMap(); - cacheKeyers = new ConcurrentHashMap>(); - - scheduleAtFixedRate.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - keysToDel.clear(); - Set keySets = cacheKeyers.keySet(); - - long currentTime = System.currentTimeMillis(); - for (K key:keySets) { - Keyer k = cacheKeyers.get(key); - long diff = currentTime-k.getLastAccessTime(); - if (diff>k.getCacheTTL()){ - keysToDel.add(k); - }else if (diff<=k.getCacheTTL() && k.getRefCount().get()>=k.getAccessCount()){ - keysToReload.add(k); - } - } - - /** - * key 失效,回调remove接口 - */ - for (Keyer key:keysToDel) { - cacheMap.remove(key.getKey()); - if(key.getRemoveKeyListener() !=null){ - key.getRemoveKeyListener().removeNotify(key.getKey(),key.getValue()); - } - cacheKeyers.remove(key.getKey()); - } - - - /** - * 回调用户reload接口 - */ - for (Keyer key:keysToReload) { - /** - * reload 设计为异步加载 - */ - if(key.getiDataLoader() !=null){ - key.getiDataLoader().reload(key); - } - } - } - },0,DEFAULT_TTL, TimeUnit.SECONDS); - } - - /** - * put (k，v) to map - * - * @param key - * @param value - * @param keyer - */ - @Override - public void put(K key, V value,Keyer keyer) { - cacheMap.put(key,value); - cacheKeyers.put(key,keyer); - } - - /** - * get value - * - * @param key - * @return - */ - @Override - public V get(K key) { - - try { - readLock.lock(); - Keyer keyer = cacheKeyers.get(key); - - if (keyer != null){ - keyer.setLastAccessTime(System.currentTimeMillis()); - keyer.getRefCount().decrementAndGet(); - } - - return cacheMap.get(key); - - }finally { - readLock.unlock(); - } - - - } - - /** - * remove key - * - * @param key - */ - @Override - public void remove(K key) { - Keyer keyer = cacheKeyers.get(key); - - if (keyer.getRemoveKeyListener()!=null){ - keyer.getRemoveKeyListener(). - removeNotify(keyer.getKey(),keyer.getValue()); - } - cacheKeyers.remove(key); - cacheMap.remove(key); - } - - - /** - * remove all map's element - */ - @Override - public void removeALL() { - for (K key:cacheKeyers.keySet()) { - Keyer keyer = cacheKeyers.get(key); - if (keyer.getRemoveKeyListener()!=null){ - keyer.getRemoveKeyListener().removeNotify(keyer.getKey(),keyer.getValue()); - } - } - cacheKeyers.clear(); - cacheMap.clear(); - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java index d44e163..9779be1 100644 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLDataLoader.java @@ -1,7 +1,10 @@ package io.mycat.sqlcache; import com.google.common.util.concurrent.*; - +import io.mycat.bigmem.sqlcache.BigSQLResult; +import io.mycat.bigmem.sqlcache.IDataLoader; +import io.mycat.bigmem.sqlcache.Keyer; +import io.mycat.bigmem.console.LocatePolicy; import java.util.concurrent.Callable; import java.util.concurrent.Executors; @@ -14,7 +17,7 @@ * @create 2017-01-20 15:13 */ -public class HintSQLDataLoader implements IDataLoader{ +public class HintSQLDataLoader implements IDataLoader { /** * 根据sql,异步从后台DB reload数据，替换旧值 * @param keyer diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java index 5fa43e6..d0375a7 100644 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/HintSQLRemoveKeyListener.java @@ -1,5 +1,8 @@ package io.mycat.sqlcache; +import io.mycat.bigmem.sqlcache.BigSQLResult; +import io.mycat.bigmem.sqlcache.IRemoveKeyListener; + /** * HintSQL 结果集被移除时回调类 * @@ -7,7 +10,7 @@ * @create 2017-01-20 15:14 */ -public class HintSQLRemoveKeyListener implements IRemoveKeyListener{ +public class HintSQLRemoveKeyListener implements IRemoveKeyListener { /** * key 失效，做清理工作 * diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java deleted file mode 100644 index 704b01b..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPage.java +++ /dev/null @@ -1,76 +0,0 @@ -package io.mycat.sqlcache; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * Cache Page接口 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-27 18:49 - */ - -public interface IBufferPage { - - /** - * get bytes from Thread Local - * - * @param position - * @param length - * @return - */ - public byte[] getBytes(int position, int length); - - /** - * ByteBuffer slice from Thread Local - * - * @param position - * @param limit - * @return - */ - public ByteBuffer slice(int position, int limit); - - /** - * get ByteBuffer from Thread Local - * - * @param position - * @return - */ - public ByteBuffer getLocalByteBuffer(int position); - - - /** - * 将数据刷到磁盘 - */ - public void flush(); - - - /** - * 页号 - * - * @return - */ - long getPageIndex(); - - - /** - * 设置页有数据被写入了，属于’脏页‘ - * - * @param dirty - */ - void setDirty(boolean dirty); - - - /** - * 页回收 - */ - public void recycle() throws IOException; - - /** - * 页是否被回收了 - * - * @return - */ - boolean isRecycled(); -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java deleted file mode 100644 index de077e7..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/IBufferPageFactory.java +++ /dev/null @@ -1,63 +0,0 @@ -package io.mycat.sqlcache; - -import java.io.IOException; -import java.util.Set; - -/** - * Buffer Page Factory 接口 - * - * @author zagnix - * @create 2016-12-27 22:59 - */ -public interface IBufferPageFactory { - - /** - * 根据index得到一页 - * - * @param index - * @return - * @throws IOException - */ - public MyCatBufferPage acquirePage(long index) throws IOException; - - /** - * 回收Page - * - * @param index - */ - public void releasePage(long index); - - /** - * 删除一页 - * @param index - * @throws IOException - */ - public void deletePage(long index) throws IOException; - - - /** - * 刷磁盘 - */ - public void flush(); - - /** - * 删除所有页 - * @throws IOException - */ - public void deleteAllPages() throws IOException; - - - /** - * 得到所有的页 - * @return - */ - public Set getExistingPageIndexSet(); - - - /** - * 从cache中移除所有的页 - * @throws IOException - */ - public void releaseCachedPages() throws IOException; - -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java deleted file mode 100644 index 87c1e7e..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/ICache.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.mycat.sqlcache; - -/** - * 定制化 Cache接口 - * 主要实现SQL结果集缓存规则 - * 1.缓存时间 - * 2.在缓存时间内，访问次数超过N次，重新从DB后台load数据，更新Value - * @author zagnix - * @version 1.0 - * @create 2016-12-30 11:26 - */ -public interface ICache { - - /** - * - * @param key - * @param value - * @param keyer - */ - public void put(final K key, final V value, final Keyer keyer); - - /** - * - * @param key - * @return - */ - public V get(final K key); - - - /** - * - * @param key - */ - public void remove(final K key); - - /** - * - */ - public void removeALL(); - - -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java deleted file mode 100644 index 9599748..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/IDataLoader.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.mycat.sqlcache; - -/** - * Value 数据加载接口 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-30 16:44 - */ -public interface IDataLoader{ - - /** - * Key失效，时候重新异步reload数据 - * @param keyer - * @return - */ - public V reload(Keyer keyer); -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java deleted file mode 100644 index f50b7a7..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/IRemoveKeyListener.java +++ /dev/null @@ -1,19 +0,0 @@ -package io.mycat.sqlcache; - -/** - * key 被移除时，回调该接口 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-30 16:47 - */ -public interface IRemoveKeyListener { - - /** - * 当Key被移除时，回调该接口 - * - * @param key - * @param value - */ - public void removeNotify(K key, V value); -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java deleted file mode 100644 index e78c886..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/ISQLResult.java +++ /dev/null @@ -1,76 +0,0 @@ -package io.mycat.sqlcache; - -import java.io.IOException; - -/** - * SQLResult 存放 实现接口 - * @author zagnix - * @create 2016-11-18 16:46 - */ -public interface ISQLResult { - - /** - * 该将sql row的二进制数据，存放到Cache中 - * @param data - * @return - * @throws IOException - */ - public long put(byte[] data) throws IOException; - - /** - * 获取一条sql row的二进制数据 - * @return - * @throws IOException - */ - public byte[] next() throws IOException; - - /** - * 根据index得到一条SQL row的二进制数据 - * @param index - * @return - * @throws IOException - */ - public byte[] get(long index) throws IOException; - - - /** - * SQL结果集条数 - * @return - */ - public long size(); - - /** - * 是否没有缓存SQL结果集 - * @return - */ - public boolean isEmpty() ; - - /** - * Cache是否已经满了 - * @return - */ - public boolean isFull(); - - /** - * 将SQL结果集刷入磁盘 - */ - public void flush(); - - /** - * 删除数据页和索引页文件 - * @throws IOException - */ - public void removeAll() throws IOException; - - /** - * 将所有内存映射文件unmap - * @throws IOException - */ - public void recycle() throws IOException; - - - /** - * 复位读的位置0，即从头开始读取SQL结果集 - */ - public void reset(); -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java deleted file mode 100644 index 7d96c46..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/Keyer.java +++ /dev/null @@ -1,113 +0,0 @@ -package io.mycat.sqlcache; - -import java.util.concurrent.atomic.AtomicLong; - -/** - * key cache生效规则 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-30 16:48 - */ - -public class Keyer { - private long cacheTTL=0; - private long lastAccessTime=0; - - /** - * TTL 时间内访问次数 - */ - private long accessCount=0; - private AtomicLong refCount = new AtomicLong(0); - - - private boolean autoRefresh = false; - - private String sql; - private K key; - private V value; - - private IDataLoader iDataLoader = null; - private IRemoveKeyListener removeKeyListener = null; - - public long getCacheTTL() { - return cacheTTL; - } - - public void setCacheTTL(long cacheTTL) { - this.cacheTTL = cacheTTL; - } - - public long getLastAccessTime() { - return lastAccessTime; - } - - public void setLastAccessTime(long lastAccessTime) { - this.lastAccessTime = lastAccessTime; - } - - public AtomicLong getRefCount() { - return refCount; - } - - public void setRefCount(AtomicLong refCount) { - this.refCount = refCount; - } - - public K getKey() { - return key; - } - - public void setKey(K key) { - this.key = key; - } - - public V getValue() { - return value; - } - - public void setValue(V value) { - this.value = value; - } - - public IDataLoader getiDataLoader() { - return iDataLoader; - } - - public void setiDataLoader(IDataLoader iDataLoader) { - this.iDataLoader = iDataLoader; - } - - public IRemoveKeyListener getRemoveKeyListener() { - return removeKeyListener; - } - - public void setRemoveKeyListener(IRemoveKeyListener removeKeyListener) { - this.removeKeyListener = removeKeyListener; - } - - public long getAccessCount() { - return accessCount; - } - - public void setAccessCount(long accessCount) { - this.accessCount = accessCount; - } - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - - public boolean isAutoRefresh() { - return autoRefresh; - } - - public void setAutoRefresh(boolean autoRefresh) { - this.autoRefresh = autoRefresh; - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java deleted file mode 100644 index 1dc0dda..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/MyCatBufferPage.java +++ /dev/null @@ -1,141 +0,0 @@ -package io.mycat.sqlcache; - - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Mycat Buffer Page 抽象类 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-27 18:49 - */ - -public abstract class MyCatBufferPage implements IBufferPage { - private final static Logger logger = - LoggerFactory.getLogger(MyCatBufferPage.class); - - protected volatile boolean dirty = false; - protected volatile boolean recycled = false; - - /** - * Thread Local Buffer Page - */ - protected ThreadLocalByteBuffer threadLocalByteBuffer; - - /** - * Page 上次访问时间 - */ - protected AtomicLong lastAccessedTimestamp; - - /** - * Page 访问引用计数 - */ - protected AtomicLong refCount = new AtomicLong(0); - - - /** - * 缓存时间 - */ - protected long cacheTTL = 0L; - - public MyCatBufferPage(ByteBuffer byteBuffer,long cacheTTL){ - this.lastAccessedTimestamp = new AtomicLong(System.currentTimeMillis()); - this.threadLocalByteBuffer = new ThreadLocalByteBuffer(byteBuffer); - this.cacheTTL = cacheTTL; - } - - - /** - * Page 是 '脏页' - * @return - */ - - public boolean isDirty() { - return dirty; - } - - - /** - * 设置 Page 是否 是 ‘脏页’ - * @param dirty - */ - public void setDirty(boolean dirty) { - this.dirty = dirty; - } - - /** - * Page 是否被 unmap了 - * @return - */ - public boolean isRecycled() { - return recycled; - } - - public void setRecycled(boolean recycled) { - this.recycled = recycled; - } - - public ThreadLocalByteBuffer getThreadLocalByteBuffer() { - return threadLocalByteBuffer; - } - - public void setThreadLocalByteBuffer(ThreadLocalByteBuffer threadLocalByteBuffer) { - this.threadLocalByteBuffer = threadLocalByteBuffer; - } - - public AtomicLong getLastAccessedTimestamp() { - return lastAccessedTimestamp; - } - - public void setLastAccessedTimestamp(AtomicLong lastAccessedTimestamp) { - this.lastAccessedTimestamp = lastAccessedTimestamp; - } - - public AtomicLong getRefCount() { - return refCount; - } - - public void setRefCount(AtomicLong refCount) { - this.refCount = refCount; - } - - public long getCacheTTL() { - return cacheTTL; - } - - public void setCacheTTL(long cacheTTL) { - this.cacheTTL = cacheTTL; - } - - - /** - * Thread Local ByteBuffer - */ - protected static class ThreadLocalByteBuffer extends ThreadLocal { - private ByteBuffer byteBuffer; - - public ThreadLocalByteBuffer(ByteBuffer src) { - byteBuffer = src; - } - - public ByteBuffer getByteBuffer() { - return byteBuffer; - } - - public void setByteBuffer(ByteBuffer byteBuffer) { - this.byteBuffer = byteBuffer; - } - - - @Override - protected synchronized ByteBuffer initialValue() { - ByteBuffer bbuffer = byteBuffer.duplicate(); - return bbuffer; - } - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java deleted file mode 100644 index 78a951f..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/PageLRUCache.java +++ /dev/null @@ -1,231 +0,0 @@ -package io.mycat.sqlcache; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * 通过LRU机制 Cache Buffer Page，提供高访问速度 - * @author zagnix - * @create 2016-12-02 14:37 - */ - -public class PageLRUCache { - private final static Logger logger = LoggerFactory.getLogger(PageLRUCache.class); - - public static final long DEFAULT_TTL = 5 * 1000; - - private final Map map; - - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private final Lock readLock = lock.readLock(); - private final Lock writeLock = lock.writeLock(); - - /** - * 异步线程 - */ - private static final ExecutorService executorService - = Executors.newCachedThreadPool(); - - private final Set toDisk = new HashSet(); - - public PageLRUCache() { - map = new HashMap(); - } - - /** - * 向Cache添加一个page，会触发Page Sweep操作 - * @param key - * @param value - * @param ttlInMilliSeconds - */ - public void put(Long key, MyCatBufferPage value, long ttlInMilliSeconds) { - Collection values = null; - try { - writeLock.lock(); - values = pageSweep(); - if (values != null && values.contains(value)) { - values.remove(value); - } - value.getLastAccessedTimestamp().set(System.currentTimeMillis()); - map.put(key, value); - } finally { - writeLock.unlock(); - } - - - if (values != null && values.size() > 0) { - executorService.execute(new Task(values)); - } - } - - public void put(Long key,MyCatBufferPage value) { - this.put(key, value, DEFAULT_TTL); - } - - - /** - * 根据Page Index 得到对应的 Page - * 同时会设置该Page的访问时间，和引用计数+1 - * @param key - * @return - */ - public MyCatBufferPage get(final Long key) { - try { - readLock.lock(); - MyCatBufferPage value = map.get(key); - if (value != null) { - value.getLastAccessedTimestamp().set(System.currentTimeMillis()); - value.getRefCount().incrementAndGet(); - } - return map.get(key); - } finally { - readLock.unlock(); - } - } - - /** - * 将上次访问时间超过Cache TTL 和 引用数=0的Page - * 存放到collection中，供put调用时候，异步从内存 - * 中刷到磁盘上 - * @return - */ - - private Collection pageSweep() { - Collection values = null; - toDisk.clear(); - Set keys = map.keySet(); - long currentTimeMillis = System.currentTimeMillis(); - - for(Long key: keys) { - MyCatBufferPage v = map.get(key); - if (v.getRefCount().get() <= 0 - && (currentTimeMillis - v.getLastAccessedTimestamp().get()) > v.getCacheTTL()) { - toDisk.add(key); - } - } - - if (toDisk.size() > 0) { - values = new HashSet(); - for(Long key : toDisk) { - MyCatBufferPage v = map.remove(key); - values.add(v); - } - } - - return values; - } - - /** - * 引用计数减1 - * @param key - */ - - public void release(final Long key) { - try { - readLock.lock(); - MyCatBufferPage value = map.get(key); - if (value != null) { - value.getRefCount().decrementAndGet(); - } - } finally { - readLock.unlock(); - } - } - - /** - * 缓存page的大小 - * @return - */ - public int size() { - try { - readLock.lock(); - return map.size(); - } finally { - readLock.unlock(); - } - } - - - /** - * 将所有缓存在内存中Page 数据，刷到磁盘中去 - * @throws IOException - */ - public void removeAll() throws IOException { - try { - writeLock.lock(); - Collection values = map.values(); - if (values != null && values.size() > 0) { - for(MyCatBufferPage v : values) { - v.recycle(); - } - } - map.clear(); - } finally { - writeLock.unlock(); - } - - } - - - /** - * 将index Page 数据，刷到磁盘中去 - * @param key - * @return - * @throws IOException - */ - - public MyCatBufferPage remove(final Long key) throws IOException { - try { - writeLock.lock(); - MyCatBufferPage value = map.remove(key); - if (value != null) { - value.recycle(); - } - return value; - } finally { - writeLock.unlock(); - } - } - - - - - public Collection getValues() { - try { - readLock.lock(); - return map.values(); - } finally { - readLock.unlock(); - } - } - - - /** - * 异步操作task，负责将内存中的页数据，刷到磁盘中 - */ - private static class Task implements Runnable { - Collection values; - public Task(Collection values) { - this.values = values; - } - public void run() { - for(MyCatBufferPage v : values) { - try { - if (v != null) { - v.recycle(); - } - } catch (IOException e) { - } - } - } - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java index fb03bcd..09c3816 100644 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java +++ b/Mycat-Core/src/main/java/io/mycat/sqlcache/SQLResultsCacheService.java @@ -6,6 +6,8 @@ import io.mycat.backend.MySQLReplicatSet; import io.mycat.backend.callback.SQLResCacheHintHandler; import io.mycat.beans.DNBean; +import io.mycat.bigmem.sqlcache.*; +import io.mycat.bigmem.console.LocatePolicy; import io.mycat.engine.UserSession; import io.mycat.front.MySQLFrontConnection; import io.mycat.mysql.MySQLConnection; @@ -62,7 +64,7 @@ public void cacheSQLResult(HintSQLInfo hintSQLInfo,BigSQLResult bigSQLResult,IDa keyer.setValue(bigSQLResult); keyer.setCacheTTL(Integer.valueOf(hintSQLInfo.getParamsKv().get("cache-time"))); keyer.setAccessCount(Integer.valueOf(hintSQLInfo.getParamsKv().get("access-count"))); - keyer.setAutoRefresh(Boolean.valueOf(hintSQLInfo.getParamsKv().get("auto-refresh"))); + // keyer.setAutoRefresh(Boolean.valueOf(hintSQLInfo.getParamsKv().get("auto-refresh"))); keyer.setRemoveKeyListener(listener); keyer.setiDataLoader(loader); sqlResultCacheImp.put(key,bigSQLResult,keyer); diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java deleted file mode 100644 index 7095773..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemoryFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -package io.mycat.sqlcache.impl.directmem; - - - -import io.mycat.sqlcache.IBufferPageFactory; -import io.mycat.sqlcache.MyCatBufferPage; - -import java.io.IOException; -import java.util.Set; - -/** - * Direct Memroy Page Factory - * - * @author zagnix - * @create 2016-12-28 07:46 - */ - -public class DirectMemoryFactory implements IBufferPageFactory { - public MyCatBufferPage acquirePage(long index) throws IOException { - return null; - } - - public void releasePage(long index) { - - } - - public void deletePage(long index) throws IOException { - - } - - public void flush() { - - } - - public void deleteAllPages() throws IOException { - - } - - public Set getExistingPageIndexSet() { - return null; - } - - public void releaseCachedPages() throws IOException { - - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java deleted file mode 100644 index 85dff63..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/directmem/DirectMemorySQLResult.java +++ /dev/null @@ -1,56 +0,0 @@ -package io.mycat.sqlcache.impl.directmem; - - -import io.mycat.sqlcache.ISQLResult; - -import java.io.IOException; - -/** - * 基于Direct Memory Cache,用queue实现 - * - * @author zagnix - * @create 2016-12-28 07:46 - */ - -public class DirectMemorySQLResult implements ISQLResult { - public long put(byte[] data) throws IOException { - return 0; - } - - public byte[] next() throws IOException { - return new byte[0]; - } - - public byte[] get(long index) throws IOException { - return new byte[0]; - } - - public long size() { - return 0; - } - - public boolean isEmpty() { - return false; - } - - public boolean isFull() { - return false; - } - - public void flush() { - - } - - public void removeAll() throws IOException { - - } - - public void recycle() throws IOException { - - } - - @Override - public void reset() { - - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java deleted file mode 100644 index 46d81da..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedBufferPageFactory.java +++ /dev/null @@ -1,307 +0,0 @@ -package io.mycat.sqlcache.impl.mmap; - - - -import io.mycat.sqlcache.IBufferPageFactory; -import io.mycat.sqlcache.MyCatBufferPage; -import io.mycat.sqlcache.PageLRUCache; -import io.mycat.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static java.nio.channels.FileChannel.MapMode.READ_WRITE; - -/** - * Memroy Mapped Page Factory - * - * @author zagnix - * @create 2016-11-18 16:46 - */ - -public class MappedBufferPageFactory implements IBufferPageFactory { - private final static Logger logger = - LoggerFactory.getLogger(MappedBufferPageFactory.class); - private int pageSize; - private String pageDirName; - private File pageDir; - private String page; - - - private final Map pageLockMap = - new ConcurrentHashMap(); - - public static final String PAGE_FILE_NAME = "page"; - public static final String PAGE_FILE_SUFFIX = ".page"; - - private final PageLRUCache pageLRUCache; - private long cacheTTL = 0L; - - - public MappedBufferPageFactory(int pageFileSize, String pageDir, long cacheTTL){ - - /** - * 页文件大小 - */ - this.pageSize = pageFileSize; - /** - * 页文件目录名 - */ - this.pageDirName = pageDir; - /** - * 页文件目录 - */ - this.pageDir = new File(this.pageDirName); - this.cacheTTL = cacheTTL; - - - if (!this.pageDir.exists()) { - this.pageDir.mkdirs(); - } - - if (!this.pageDirName.endsWith(File.separator)) { - this.pageDirName += File.separator; - } - /** - * 页文件名前缀 - */ - this.page = this.pageDirName + PAGE_FILE_NAME + "-"; - this.pageLRUCache = new PageLRUCache(); - - } - - /** - * 查询page - * @param index - * @return - * @throws IOException - */ - public MyCatBufferPage acquirePage(long index) throws IOException { - /** - * 首先从Cache中查找对应的页 - */ - MyCatBufferPage myCatBufferPage = pageLRUCache.get(index); - - if (myCatBufferPage == null) { - try { - if (!pageLockMap.containsKey(index)) { - pageLockMap.put(index,new ReentrantReadWriteLock()); - } - Lock writeLock = pageLockMap.get(index).writeLock(); - /** - * 加锁处理当前index - */ - try { - writeLock.lock(); - /** - * 可能其他线程已经加入到了cache中了。 - */ - myCatBufferPage = pageLRUCache.get(index); - - if (myCatBufferPage == null) { - /** - * 创建一个页，利用Java MappedByteBuffer 高性能内存映射 - */ - RandomAccessFile raf = null; - FileChannel channel = null; - try { - String pageName = this.getPageNameByIndex(index); - raf = new RandomAccessFile(pageName, "rw"); - channel = raf.getChannel(); - - /** - * 文件映射 转换成 MappedByteBuffer - */ - MappedByteBuffer mbb = channel.map(READ_WRITE,0,this.pageSize); - - /** - * 转换成逻辑的映射页 - */ - myCatBufferPage = new MappedMemFileBufferPage(mbb, pageName, index,cacheTTL); - - /** - * 缓存起来。。。lRU Cache....... - */ - pageLRUCache.put(index, myCatBufferPage); - - if (logger.isDebugEnabled()) { - logger.debug("Mapped page for " + pageName + " was just created and cached."); - } - - } finally { - if (channel != null) channel.close(); - if (raf != null) raf.close(); - } - } - }finally { - writeLock.unlock(); - } - } finally { - pageLockMap.remove(index); - } - } - return myCatBufferPage; - } - - /** - * 根据索引生成页文件 - * @param index - * @return - */ - private String getPageNameByIndex(long index) { - return this.page + index + PAGE_FILE_SUFFIX; - } - - /** - * 回收Page - * @param index - */ - public void releasePage(long index) { - pageLRUCache.release(index); - } - - /** - * 通过page文件名,返回其page索引 - * @param pageName - * @return - */ - private long getIndexByPageName(String pageName) { - int beginIndex = pageName.lastIndexOf('-'); - beginIndex += 1; - int endIndex = pageName.lastIndexOf(PAGE_FILE_SUFFIX); - String sIndex = pageName.substring(beginIndex, endIndex); - long index = Long.parseLong(sIndex); - return index; - } - - - /** - * 根据索引删除页文件，调用者需要同步访问 - * @param index - * @throws IOException - */ - public void deletePage(long index) throws IOException { - pageLRUCache.remove(index); - String pageName = this.getPageNameByIndex(index); - int count = 0; - int maxRound = 10; - boolean deleted = false; - while(count < maxRound) { - try { - Utils.deleteFile(new File(pageName)); - deleted = true; - break; - } catch (IllegalStateException ex) { - try { - Thread.sleep(200); - } catch (InterruptedException e) { - } - count++; - if (logger.isDebugEnabled()) { - logger.warn("fail to delete " + pageName + ", tried round = " + count); - } - } - } - if (deleted) { - if(logger.isDebugEnabled()) { - logger.debug("Page " + pageName + " was just deleted."); - } - } else { - logger.warn("fail to delete " + pageName + " after max " + maxRound + " rounds of try, you may delete it manually."); - } - } - - - /** - * thread unsafe, caller need synchronization - */ - public void flush() { - Collection cachedPages = pageLRUCache.getValues(); - for(MyCatBufferPage mappedPage : cachedPages) { - ((MappedMemFileBufferPage)mappedPage).flush(); - } - } - - /** - * thread unsafe, caller need synchronization - */ - public void deleteAllPages() throws IOException { - pageLRUCache.removeAll(); - Set indexSet = getExistingPageIndexSet(); - this.deletePages(indexSet); - } - - /** - * thread unsafe, caller need synchronization - */ - public void deletePages(Set indexes) throws IOException { - if (indexes == null) return; - for(long index : indexes) { - this.deletePage(index); - } - } - - public Set getExistingPageIndexSet() { - Set indexSet = new HashSet(); - File[] pageFiles = this.pageDir.listFiles(); - if (pageFiles != null && pageFiles.length > 0) { - for(File pageFile : pageFiles) { - String fileName = pageFile.getName(); - if (fileName.endsWith(PAGE_FILE_SUFFIX)) { - long index = this.getIndexByPageName(fileName); - indexSet.add(index); - } - } - } - return indexSet; - } - - public void releaseCachedPages() throws IOException { - pageLRUCache.removeAll(); - } - - - public int getPageSize() { - return pageSize; - } - - public void setPageSize(int pageSize) { - this.pageSize = pageSize; - } - - public String getPageDirName() { - return pageDirName; - } - - public void setPageDirName(String pageDirName) { - this.pageDirName = pageDirName; - } - - public File getPageDir() { - return pageDir; - } - - public void setPageDirFile(File pageDir) { - this.pageDir = pageDir; - } - - public String getPage() { - return page; - } - - public void setPageFile(String page) { - this.page = page; - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java deleted file mode 100644 index f63640b..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedMemFileBufferPage.java +++ /dev/null @@ -1,193 +0,0 @@ -package io.mycat.sqlcache.impl.mmap; - - -import io.mycat.sqlcache.MyCatBufferPage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.security.AccessController; -import java.security.PrivilegedAction; - -/** - * 内存映射文件实现data存储 - * - * @author zagnix - * @version 1.0 - * @create 2016-12-27 18:52 - */ - -public class MappedMemFileBufferPage extends MyCatBufferPage { - private final static Logger logger = - LoggerFactory.getLogger(MappedMemFileBufferPage.class); - private String pageName = null; - private long pageIndex = 0L; - private static Method mappedByteBufferCleaner = null; - private static Method mappedByteBufferCleanerClean = null; - - static { - try { - mappedByteBufferCleaner = - Class.forName("java.nio.DirectByteBuffer").getMethod("cleaner"); - mappedByteBufferCleaner.setAccessible(true); - mappedByteBufferCleanerClean = - Class.forName("sun.misc.Cleaner").getMethod("clean"); - mappedByteBufferCleanerClean.setAccessible(true); - } catch (NoSuchMethodException e) { - e.printStackTrace(); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - - - /** - * 创建 一个MappedByteBufferPage对象 - * - * @param mappedByteBuffer - * @param file - * @param index - * @param cacheTTL - */ - public MappedMemFileBufferPage(MappedByteBuffer mappedByteBuffer, String file, long index, long cacheTTL){ - super(mappedByteBuffer.load(),cacheTTL); - this.pageName = file; - this.pageIndex = index; - } - - - /** - * - * @param position - * @param length - * @return - */ - public byte[] getBytes(int position, int length) { - ByteBuffer buf = this.getLocalByteBuffer(position); - byte[] data = new byte[length]; - buf.get(data); - return data; - } - - /** - * ByteBuffer slice from Thread Local - * @param position - * @param limit - * @return - */ - public ByteBuffer slice(int position, int limit) { - ByteBuffer buffer = this.threadLocalByteBuffer.get(); - buffer.limit(position+limit); - buffer.position(position); - return buffer.slice(); - } - - /** - * ByteBuffer from Thread Local - * - * @param position - * @return - */ - public ByteBuffer getLocalByteBuffer(int position) { - ByteBuffer buf = this.threadLocalByteBuffer.get(); - buf.position(position); - return buf; - } - - - /** - * 将pageName的数据刷入磁盘 - */ - public void flush() { - synchronized(this) { - if (this.recycled) return; - if (dirty) { - MappedByteBuffer mappedByteBuffer = - (MappedByteBuffer)threadLocalByteBuffer.getByteBuffer(); - mappedByteBuffer.force(); - dirty = false; - } - } - - } - - /** - * 返回 页 号 - * @return - */ - public long getPageIndex() { - return this.pageIndex; - } - - /** - * 设置 页 为 dirty - * @param dirty - */ - public void setDirty(boolean dirty) { - this.dirty = dirty; - } - - /** - * 回收页，关闭文件pageName - * @throws IOException - */ - public void recycle() throws IOException{ - - synchronized(this) { - if (this.recycled) return; - flush(); - MappedByteBuffer srcBuf = - (MappedByteBuffer)threadLocalByteBuffer.getByteBuffer(); - unmap(srcBuf); - this.threadLocalByteBuffer = null; - this.recycled = true; - } - - } - - /** - * 解文件内存映射 - * - * @param buffer - */ - private void unmap(final MappedByteBuffer buffer) { - if(buffer == null) - return; - - AccessController.doPrivileged(new PrivilegedAction() { - public Object run() { - try { - if (mappedByteBufferCleaner != null && - mappedByteBufferCleanerClean != null && - buffer.isDirect()) { - Object cleaner = mappedByteBufferCleaner.invoke(buffer); - mappedByteBufferCleanerClean.invoke(cleaner); - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - return null; - } - }); - } - - - public boolean isRecycled() { - return false; - } - - public String getPageName() { - return pageName; - } - - public void setPageName(String pageName) { - this.pageName = pageName; - } - - public void setPageIndex(long pageIndex) { - this.pageIndex = pageIndex; - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java deleted file mode 100644 index 5f80355..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/impl/mmap/MappedSQLResult.java +++ /dev/null @@ -1,890 +0,0 @@ -package io.mycat.sqlcache.impl.mmap; - - -import io.mycat.sqlcache.AddressIndex; -import io.mycat.sqlcache.ISQLResult; -import io.mycat.sqlcache.MyCatBufferPage; -import io.mycat.util.UnsafeMemory; -import io.mycat.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * 基于内存映射Cache,用queue实现 - * - * @author zagnix - * @create 2016-12-02 15:46 - */ -public class MappedSQLResult implements ISQLResult { - private final static Logger LOGGER = LoggerFactory.getLogger(MappedSQLResult.class); - /** - * 存放索引文件的目录 - */ - private static final String INDEX_PAGE_FOLDER = "index"; - /** - * 存放数据的目录 - */ - private static final String DATA_PAGE_FOLDER = "data"; - /** - * 保存元数据信息目录 - */ - private static final String META_DATA_PAGE_FOLDER = "meta_data"; - /** - * Page File Size - */ - private final int DATA_PAGE_SIZE; - - /** - * 默认页文件大小 - */ - public final static int DEFAULT_DATA_PAGE_SIZE = 128 * 1024 * 1024; - - /** - * 最小页文件大小 - */ - public final static int MINIMUM_DATA_PAGE_SIZE = 8 * 1024 * 1024; - - /** - * 索引页在内存中停留时间 - */ - public final static int INDEX_PAGE_CACHE_TTL = 1000; - /** - * 数据也在内存中停留时间 - */ - public final static int DATA_PAGE_CACHE_TTL = 1000; - /** - *索引文件地址编码,使用 64bit, - * 一条索引地址格式为： - * page number | page offset | data lenghe | op - * 63 50 | 49 21 | 20 1| 0 - */ - /** - * 最大data page 文件数16 * 1024 - */ - public final static int DATA_PAGE_NUMBER_BITS = 14; - /** - * 每个data文件大小最大为512M - */ - public final static int DATA_PAGE_INDEX_ITEM_OFFSET_BITS = 29; - /** - * 每次添加的数据最大长度1M - */ - public final static int DATA_ITEM_LEN_BITS = 20; - - /** - * 标志是否被消费了。用于多线程并发访问标志 - */ - public final static int DATA_PAGE_INDEX_OP_BITS = 1; - - /** - * 最大DATA页大小 - */ - public final static int MAX_DATA_PAGE_SIZE = 1 << DATA_PAGE_INDEX_ITEM_OFFSET_BITS; - /** - *最大DATA页的数量 - */ - public final static int MAX_DATA_PAGE_INDEX_SIZE = 1 << DATA_PAGE_NUMBER_BITS; - /** - * 每条item data最大长度 - */ - public final static int MAX_DATA_ITEM_LEN = 1 << DATA_ITEM_LEN_BITS; - /** - * 位操作用的 - */ - private static final long MASK_LONG_14_BITS = 0xFFF3L; - private static final long MASK_LONG_29_BITS = 0x1FFFFFFFL; - private static final long MASK_LONG_20_BITS = 0xFFFFFL; - private static final long MASK_LONG_1_BITS = 0x1L; - - /** - * 存放页文件目录 - */ - private String cacheDirectory = null; - - /** - * 索引页管理对象，提供acquire/release/cache - */ - private MappedBufferPageFactory indexPageFactory = null; - - /** - * 数据页管理对象，提供acquire/release/cache - */ - private MappedBufferPageFactory dataPageFactory = null; - - /** - * 元数据管理对象，提供acquire/release/cache - */ - private MappedBufferPageFactory metaPageFactory = null; - - private static final int META_DATA_PAGE_INDEX = 0; - private static final int META_DATA_ITEM_LENGTH_BITS = 4; - private static final int META_DATA_PAGE_SIZE = 1 << META_DATA_ITEM_LENGTH_BITS; - - /** - * index页存放data item数量是1024 * 1024条 - */ - private static final int INDEX_ITEMS_PER_PAGE_BITS = 20; - private static final int INDEX_ITEMS_PER_PAGE = 1 << INDEX_ITEMS_PER_PAGE_BITS; - /** - * 每条data item的编码 index address的大小为8bit - */ - private static final int INDEX_ITEM_LENGTH_BITS = 3; - private static final int INDEX_ITEM_LENGTH = 1 << INDEX_ITEM_LENGTH_BITS; - /** - * index 页文件大小为8*1024*1024 - */ - private static final int INDEX_PAGE_SIZE = INDEX_ITEM_LENGTH * INDEX_ITEMS_PER_PAGE; - - /** - * 队列头下标 - */ - private final AddressIndex queueFrontIndex = new AddressIndex(); - /** - * 队列尾下标 - */ - private final AddressIndex queueTailIndex = new AddressIndex(); - /** - * queueTailIndex下标 索引的data page - */ - private final AddressIndex queueTailDataPageIndex = new AddressIndex(); - - /** - * queueTailIndex下标索引的data page的当前写位置 - */ - private final AddressIndex queueTailDataItemOffset = new AddressIndex(); - - /** - * lock - */ - private final Lock appendLock = new ReentrantLock(); - private final ReadWriteLock arrayReadWritelock = new ReentrantReadWriteLock(); - private final Lock queueReadLock = arrayReadWritelock.readLock(); - private final Lock queueWriteLock = arrayReadWritelock.writeLock(); - - - - /** - * MappedBigCache构造函数 - * @param cacheDir - * @param cacheName - * @throws IOException - */ - public MappedSQLResult(String cacheDir, String cacheName) throws IOException{ - this(cacheDir,cacheName,DEFAULT_DATA_PAGE_SIZE); - } - - /** - * MappedSQLResult 构造函数 - * @param cacheDir - * @param cacheName - * @param cacheSize - * @throws IOException - */ - public MappedSQLResult(String cacheDir, String cacheName, int cacheSize) throws IOException { - - this.cacheDirectory = cacheDir; - /** - * 目录没有加文件分割符 - */ - if (!cacheDirectory.endsWith(File.separator)) { - cacheDirectory += File.separator; - } - - /** - * 构建一个消息队列名字 - */ - cacheDirectory = cacheDirectory + cacheName + File.separator; - - /** - * 确定路径合法 - */ - if (!Utils.isFilenameValid(cacheDirectory)) { - throw new IllegalArgumentException("invalid array directory : " + cacheDirectory); - } - - long pageSize = UnsafeMemory.roundToOsPageSzie((long)cacheSize); - - if (pageSize < MINIMUM_DATA_PAGE_SIZE) { - throw new IllegalArgumentException("invalid page size, allowed minimum is : " + MINIMUM_DATA_PAGE_SIZE + " bytes."); - } - - if (pageSize > MAX_DATA_PAGE_SIZE){ - throw new IllegalArgumentException("invalid page size, allowed max is : " + MAX_DATA_PAGE_SIZE + " bytes."); - } - - DATA_PAGE_SIZE = (int)pageSize; - this.init(); - } - - - private void init() throws IOException { - /** - * 索引页管理对象构建 - */ - this.indexPageFactory = new MappedBufferPageFactory(INDEX_PAGE_SIZE, - this.cacheDirectory + INDEX_PAGE_FOLDER, - INDEX_PAGE_CACHE_TTL); - - /** - * 数据页管理对象构建 - */ - this.dataPageFactory = new MappedBufferPageFactory(DATA_PAGE_SIZE, - this.cacheDirectory + DATA_PAGE_FOLDER, - DATA_PAGE_CACHE_TTL); - - /** - * 元信息管理对象构建 - */ - this.metaPageFactory = new MappedBufferPageFactory(META_DATA_PAGE_SIZE, - this.cacheDirectory + META_DATA_PAGE_FOLDER, - 10 * 1000); - - /** - * 初始化数组head 和 tail下标 - */ - initArrayIndex(); - - /** - * 初始化当前页号和偏移量 - */ - initDataPageIndex(); - - } - - - /** - * 初始化Queue的 front index 和 tail index - * @throws IOException - */ - void initArrayIndex() throws IOException { - /** - * 从元数据信息中获取数组的head / tail index - */ - MyCatBufferPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); - ByteBuffer metaBuf = metaDataPage.getLocalByteBuffer(0); - - long front= metaBuf.getLong(); - long tail = metaBuf.getLong(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(" head " + front); - LOGGER.debug(" tail " + tail); - } - /** - * head .tail 是保存在文件中， - * 程序启动时候，会读取数据元文件的head，tail信息初始化一个数组的head和tail - */ - queueFrontIndex.set(front); - queueTailIndex.set(tail); - } - - /** - * 初始化当前页号和偏移量 - * @throws IOException - */ - void initDataPageIndex() throws IOException { - /** - * 数组为空的情况 - */ - if (this.isEmpty()) { - queueTailDataPageIndex.set(0); - queueTailDataItemOffset.set(0); - } else { - - MyCatBufferPage previousIndexPage = null; - long previousIndexPageIndex = -1; - - try { - /** - * 根据数组的信息初始化Data Page的写位置 = tailIndex -1 - */ - long previousIndex = this.queueTailIndex.get() - 1; - - if (previousIndex < 0) { - previousIndex = Long.MAX_VALUE; - } - - /** - * previousIndex通过取模找到previousIndex所在的 index page - */ - previousIndexPageIndex = previousIndex / (INDEX_ITEMS_PER_PAGE); - - if (LOGGER.isDebugEnabled()) { - LOGGER.info("====>previousIndexPageIndex " + previousIndexPageIndex); - } - - /** - * MemoryMappedFile Index Page - */ - previousIndexPage = this.indexPageFactory.acquirePage(previousIndexPageIndex); - - /** - * 当前 index page内写位置 - */ - int previousIndexPageOffset = (int)(previousIndex * INDEX_ITEM_LENGTH); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("====>previousIndexPageOffset " + previousIndex); - } - - /** - * 根据页内偏移的位置得到对应的ByteBuffer - */ - ByteBuffer previousIndexItemBuffer = previousIndexPage.getLocalByteBuffer(previousIndexPageOffset); - - /** - * data index address - * data page number|data page offset|data len|op - */ - long previousDataPageIndexAddress = previousIndexItemBuffer.getLong(); - - /** - * 整个Data Page中当前的页号。 - * 以及Page内当前的偏移量。 - */ - queueTailDataPageIndex.set(decodeDataPageNumber(previousDataPageIndexAddress)); - queueTailDataItemOffset.set(decodeDataPageCurrentOffset(previousDataPageIndexAddress) + - decodeDataLen(previousDataPageIndexAddress)); - } finally { - /** - * 释放当前页 - */ - if (previousIndexPage != null) { - this.indexPageFactory.releasePage(previousIndexPageIndex); - } - } - } - } - /** - * 把data存到数据中 - * @param data - * @return - * @throws IOException - */ - public long put(byte[] data) throws IOException { - try { - queueWriteLock.lock(); - - /** - * 数据长度超过1MB的情况 - */ - if (data.length > MAX_DATA_ITEM_LEN){ - throw new IOException("data length larger :" + MAX_DATA_ITEM_LEN); - } - - /** - * 数据页 - */ - MyCatBufferPage toAppendDataPage = null; - - /** - * 索引页 - */ - MyCatBufferPage toAppendIndexPage = null; - - /** - * 追加的索引下标.... - */ - long toAppendIndexPageIndex = -1L; - - /** - * 追加数据索引下标.... - */ - long toAppendDataPageIndex = -1; - long toAppendArrayIndex = -1L; - - try { - /** - * 锁住当前，只允许一个线程访问 - */ - appendLock.lock(); // only one thread can append - - /** - * 数据是否达到最大空间极限 - */ - if (this.isFull()) { // end of the world check:) - throw new IOException("ring space of java long type used up, the end of the world!!!"); - } - /** - * 大于Data Page的大小,跳到下一页 - */ - if (this.queueTailDataItemOffset.get() + data.length > DATA_PAGE_SIZE) { - /** - * 整个索引下标超过了MAX_DATA_PAGE_INDEX_SIZE，回到下标为0 - */ - if (this.queueTailDataPageIndex.get() == (MAX_DATA_PAGE_INDEX_SIZE)) { - this.queueTailDataPageIndex.set(0L); - } else { - this.queueTailDataPageIndex.incrementAndGet(); - } - /** - * 新的页内偏移设置0。 - */ - this.queueTailDataItemOffset.set(0L); - } - - /** - * 还是在当前页中直接赋值。。。。。 - */ - toAppendDataPageIndex = this.queueTailDataPageIndex.get(); - int toAppendDataItemOffset = (int)this.queueTailDataItemOffset.get(); - - /** - * 队列尾下标 - */ - toAppendArrayIndex = this.queueTailIndex.get(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("toAppendArrayIndex " + toAppendArrayIndex); - } - - /** - * 查找toAppendDataPageIndex对应的MemoryMappedFile对象 - */ - toAppendDataPage = this.dataPageFactory.acquirePage(toAppendDataPageIndex); - - /** - * 根据toAppendDataItemOffset此时的读写偏移位置返回一个ByteBuffer对象，并设置position - * 的位置为 toAppendDataItemOffset - */ - ByteBuffer toAppendDataPageBuffer = toAppendDataPage.getLocalByteBuffer(toAppendDataItemOffset); - - /** - * 将item的数据写入ByteBuffer中 - */ - toAppendDataPageBuffer.put(data); - - /** - * 当前页有数据了，标志为脏页 - */ - toAppendDataPage.setDirty(true); - - /** - * 更新当前page页面的数据item偏移位置。 - */ - this.queueTailDataItemOffset.addAndGet(data.length); - - /** - * 将数据页号和当前数据数据页号编码成 index address存放到索引页中 - */ - - toAppendIndexPageIndex = toAppendArrayIndex / (INDEX_ITEMS_PER_PAGE); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("index page number: " + toAppendIndexPageIndex); - } - - toAppendIndexPage = this.indexPageFactory.acquirePage(toAppendIndexPageIndex); - int toAppendIndexItemOffset = (int) ((toAppendArrayIndex&(INDEX_ITEMS_PER_PAGE-1))*INDEX_ITEM_LENGTH); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("index page offset: " + toAppendIndexItemOffset); - } - - ByteBuffer toAppendIndexPageBuffer = toAppendIndexPage.getLocalByteBuffer(toAppendIndexItemOffset); - - /** - * 就是记录data item 所在的数据页号，数据页偏移位置，item数据长度 - */ - toAppendIndexPageBuffer.putLong(encodeIndexAddress((int)toAppendDataPageIndex,toAppendDataItemOffset,data.length,(byte) 0x1)); - toAppendIndexPage.setDirty(true); - /** - * 将数据的头加1 - */ - this.queueTailIndex.incrementAndGet(); - - /** - * 更新元数据,只有一页的0 - * 保存当前的数组的头和尾信息 - */ - MyCatBufferPage metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); - ByteBuffer metaDataBuf = metaDataPage.getLocalByteBuffer(0); - metaDataBuf.putLong(this.queueFrontIndex.get()); - metaDataBuf.putLong(this.queueTailIndex.get()); - metaDataPage.setDirty(true); - - } finally { - - appendLock.unlock(); - - if (toAppendDataPage != null) { - this.dataPageFactory.releasePage(toAppendDataPageIndex); - } - if (toAppendIndexPage != null) { - this.indexPageFactory.releasePage(toAppendIndexPageIndex); - } - } - - return toAppendArrayIndex; - - } finally { - queueWriteLock.unlock(); - } - } - - /** - * 取数据 - * @return - * @throws IOException - */ - public byte[] next() throws IOException { - try { - queueWriteLock.lock(); - - if (this.isEmpty()) { - return null; - } - - long frontIndex = this.queueFrontIndex.get(); - - - MyCatBufferPage dataPage = null; - long dataItemIndexAddress = -1L; - int dataPageNumber = -1; - try { - /** - * 根据index，获取index address。 - */ - ByteBuffer indexItemBuffer = this.getIndexItemBuffer(frontIndex); - - /** - * 得到data item 的索引地址，通过索引地址解码出页号，页偏移，data Item 的长度 - */ - dataItemIndexAddress = indexItemBuffer.getLong(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("dataItemIndexAddress " + dataItemIndexAddress); - } - - dataPageNumber = decodeDataPageNumber(dataItemIndexAddress); - int dataItemOffset = decodeDataPageCurrentOffset(dataItemIndexAddress); - int dataItemLength = decodeDataLen(dataItemIndexAddress); - - dataPage = this.dataPageFactory.acquirePage(dataPageNumber); - - byte[] data = dataPage.getBytes(dataItemOffset,dataItemLength); - - long nextQueueFrontIndex = frontIndex; - if (nextQueueFrontIndex == Long.MAX_VALUE) { - nextQueueFrontIndex = 0L; - } else { - nextQueueFrontIndex++; - } - this.queueFrontIndex.set(nextQueueFrontIndex); - MyCatBufferPage queueFrontIndexPage = - this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); - ByteBuffer queueFrontIndexBuffer = - queueFrontIndexPage.getLocalByteBuffer(0); - queueFrontIndexBuffer.putLong(queueFrontIndex.get()); - queueFrontIndexBuffer.putLong(queueTailIndex.get()); - queueFrontIndexPage.setDirty(true); - return data; - } finally { - if (dataPage != null) { - this.dataPageFactory.releasePage(dataPageNumber); - } - } - } finally { - queueWriteLock.unlock(); - } - } - - - /** - *get下标为index的数据 - * @param index - * @return - * @throws IOException - */ - public byte[] get(long index) throws IOException { - try { - queueReadLock.lock(); - validateIndex(index); - MyCatBufferPage dataPage = null; - long dataItemIndexAddress = -1L; - int dataPageNumber = -1; - try { - - /** - * 根据index，获取index address。 - */ - ByteBuffer indexItemBuffer = this.getIndexItemBuffer(index); - - /** - * 得到data item 的索引地址，通过索引地址解码出页号，页偏移，data Item 的长度 - */ - dataItemIndexAddress = indexItemBuffer.getLong(); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("dataItemIndexAddress " + dataItemIndexAddress); - } - - dataPageNumber = decodeDataPageNumber(dataItemIndexAddress); - int dataItemOffset = decodeDataPageCurrentOffset(dataItemIndexAddress); - int dataItemLength = decodeDataLen(dataItemIndexAddress); - - dataPage = this.dataPageFactory.acquirePage(dataPageNumber); - - byte[] data = dataPage.getBytes(dataItemOffset,dataItemLength); - - return data; - } finally { - if (dataPage != null) { - this.dataPageFactory.releasePage(dataPageNumber); - } - } - } finally { - queueReadLock.unlock(); - } - } - - /** - * 队列大小 - * @return - */ - public long size() { - long qFront = this.queueFrontIndex.get(); - long qRear = this.queueTailIndex.get(); - if (qFront <= qRear) { - return (qRear - qFront); - } else { - return Long.MAX_VALUE - qFront + 1 + qRear; - } - } - - /** - * 确定index是否合法 - * @param index - */ - - void validateIndex(long index) { - if (this.queueFrontIndex.get() <= this.queueTailIndex.get()) { - if (index < this.queueFrontIndex.get() || index >= this.queueTailIndex.get()) { - throw new IndexOutOfBoundsException(); - } - } else { - if (index < this.queueFrontIndex.get() && index >= this.queueTailIndex.get()) { - throw new IndexOutOfBoundsException(); - } - } - } - - /** - * 从索引文件中找到数组下标index对应的索引ByteBuffer - * @param index - * @return - * @throws IOException - */ - ByteBuffer getIndexItemBuffer(long index) throws IOException { - - MyCatBufferPage indexPage = null; - long indexPageIndex = -1L; - try { - - indexPageIndex = index / (INDEX_ITEMS_PER_PAGE); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("get index " + indexPageIndex); - } - - indexPage = this.indexPageFactory.acquirePage(indexPageIndex); - int indexItemOffset = (int) ((index&(INDEX_ITEMS_PER_PAGE-1)) * INDEX_ITEM_LENGTH); - ByteBuffer indexItemBuffer = indexPage.getLocalByteBuffer(indexItemOffset); - return indexItemBuffer; - - } finally { - if (indexPage != null) { - this.indexPageFactory.releasePage(indexPageIndex); - } - } - } - - /** - * 将Data页号，以及当前可写偏移位置，写入data的长度，以及op位编码成一个64为的地址 - * @param dataPageNumber - * @param currentOffset - * @param dataLen - * @param op - * @return - */ - public static long encodeIndexAddress(int dataPageNumber, - int currentOffset,int dataLen,byte op) { - - return ((long)dataPageNumber)<<(DATA_PAGE_INDEX_ITEM_OFFSET_BITS+DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS)| - ((long)currentOffset)<<(DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS)| - ((long)dataLen)<>> - (DATA_PAGE_INDEX_ITEM_OFFSET_BITS + - DATA_ITEM_LEN_BITS+ - DATA_PAGE_INDEX_OP_BITS))&MASK_LONG_14_BITS); - } - - /** - * 从索引地址中分离出当前data页写的位置 - * @param indexAddress - * @return - */ - private static int decodeDataPageCurrentOffset(long indexAddress) { - return (int)((indexAddress >>> (DATA_ITEM_LEN_BITS+DATA_PAGE_INDEX_OP_BITS))&MASK_LONG_29_BITS); - } - - /** - * 从索引地址分离出当前data页的存放数据长度 - * @param indexAddress - * @return - */ - private static int decodeDataLen(long indexAddress) { - return (int)((indexAddress >>> DATA_PAGE_INDEX_OP_BITS)&MASK_LONG_20_BITS); - } - - /** - * 从索引地址分离出最低位的值 - * @param indexAddress - * @return - */ - private static byte decodeOpBit(long indexAddress) { - return (byte) (indexAddress&MASK_LONG_1_BITS); - } - - - /** - * 判断队列是否为空？ - * @return - */ - public boolean isEmpty() { - try { - queueReadLock.lock(); - return this.queueFrontIndex.get() == this.queueTailIndex.get(); - } finally { - queueReadLock.unlock(); - } - } - - /** - * 判断队列是否满了 - * @return - */ - public boolean isFull() { - try { - queueReadLock.lock(); - long currentIndex = this.queueTailIndex.get(); - long nextIndex = currentIndex == (Long.MAX_VALUE) ? 0 : currentIndex + 1; - return nextIndex == this.queueFrontIndex.get(); - } finally { - queueReadLock.unlock(); - } - } - - - /** - * 队头下标 - * @return - */ - public long getQueueFrontIndex() { - try { - queueReadLock.lock(); - return queueFrontIndex.get(); - } finally { - queueReadLock.unlock(); - } - } - - /** - * 队尾下标 - * @return - */ - public long getQueueTailIndex() { - try { - queueReadLock.lock(); - return queueTailIndex.get(); - } finally { - queueReadLock.unlock(); - } - } - - - - /** - * 刷页面到磁盘总 - */ - public void flush() { - try { - queueReadLock.lock(); - this.metaPageFactory.flush(); - this.indexPageFactory.flush(); - this.dataPageFactory.flush(); - } finally { - queueReadLock.unlock(); - } - } - - /** - * 从cache中移走，并unmap操作 - * @throws IOException - */ - public void recycle() throws IOException { - try { - queueWriteLock.lock(); - if (this.metaPageFactory != null) { - this.metaPageFactory.releaseCachedPages(); - } - if (this.indexPageFactory != null) { - this.indexPageFactory.releaseCachedPages(); - } - if (this.dataPageFactory != null) { - this.dataPageFactory.releaseCachedPages(); - } - } finally { - queueWriteLock.unlock(); - } - } - - /** - * reset front index = 0 - */ - public void reset(){ - this.queueFrontIndex.set(0); - MyCatBufferPage metaDataPage = null; - try { - metaDataPage = this.metaPageFactory.acquirePage(META_DATA_PAGE_INDEX); - } catch (IOException e) { - e.printStackTrace(); - } - ByteBuffer metaDataBuf = metaDataPage.getLocalByteBuffer(0); - metaDataBuf.putLong(this.queueFrontIndex.get()); - metaDataBuf.putLong(this.queueTailIndex.get()); - metaDataPage.setDirty(true); - } - - /** - * 删除所有文件并初始化环境 - * @throws IOException - */ - public void removeAll() throws IOException { - try { - queueWriteLock.lock(); - this.indexPageFactory.deleteAllPages(); - this.dataPageFactory.deleteAllPages(); - this.metaPageFactory.deleteAllPages(); - this.init(); - } finally { - queueWriteLock.unlock(); - } - } -} diff --git a/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java b/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java deleted file mode 100644 index 5fee33d..0000000 --- a/Mycat-Core/src/main/java/io/mycat/sqlcache/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -package io.mycat.sqlcache; - -/** - * - * testCase: - * \test\java\io\mycat\bigmem\sqlcache\testBigSQLResultCache.java - * */ \ No newline at end of file From 35dbfd7a2a7a5723d28a223385e276ada3e1bca7 Mon Sep 17 00:00:00 2001 From: zagnix Date: Tue, 28 Feb 2017 21:27:22 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B5=8B=E8=AF=95Case?= =?UTF-8?q?=E6=B2=A1=E6=9C=89import=20bigmemory=E7=9B=B8=E5=85=B3=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java | 5 +++-- .../src/test/java/io/mycat/sqlcache/testSQLResultCache.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java b/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java index f1c87e6..439ba2e 100644 --- a/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java +++ b/Mycat-Core/src/test/java/io/mycat/sqlcache/MyCatBigSqlResultsCache.java @@ -1,8 +1,9 @@ package io.mycat.sqlcache; import com.google.common.util.concurrent.*; +import io.mycat.bigmem.sqlcache.*; import io.mycat.util.Utils; - +import io.mycat.bigmem.console.LocatePolicy; import java.util.concurrent.Callable; import java.util.concurrent.Executors; @@ -38,7 +39,7 @@ private MyCatBigSqlResultsCache(){ * @param listener key被移除时，调用的接口 */ public void cacheSQLResult(String sql, BigSQLResult bigSQLResult, long cache, long accesCount, - IDataLoader loader, IRemoveKeyListener listener){ + IDataLoader loader, IRemoveKeyListener listener){ /** * TODO */ diff --git a/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java b/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java index c0d4263..d13d6cb 100644 --- a/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java +++ b/Mycat-Core/src/test/java/io/mycat/sqlcache/testSQLResultCache.java @@ -1,6 +1,8 @@ package io.mycat.sqlcache; +import io.mycat.bigmem.console.LocatePolicy; +import io.mycat.bigmem.sqlcache.BigSQLResult; import io.mycat.util.Utils; import org.junit.After; import org.junit.Assert;